#!/bin/bash
echo -e "Starting Dashboards alerts and dashboard generation"

# It takes some time to get domain created, and gives Access denied error. Adding a sleep of 60 second so as domain gets created before running the POST commands.
sleep 60;
curl -s -XGET -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT'
sleep 10;

# There was a bug in elastic which can't store the origin url when string field is converted to URL type and it is exported to another domain. Workaround is to replace it manually.
# https://github.com/elastic/kibana/issues/63924
InstanceIP=`curl ifconfig.me`
sed -i 's/CHANGE_ORIGIN_URL/'$InstanceIP'/g' /home/ec2-user/assets/export_opensearch_dashboards_V1_0.ndjson

# Create backend role to load CW logs using lambda
curl -s -XPATCH -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_security/api/rolesmapping/all_access' -H 'Content-Type: application/json' -d '[ {"op":"add","path":"/backend_roles","value":["LAMBDA_CW_LOGS_ROLE_ARN"]} ] '

# Generate auth for Default Dashboards
curl -XPOST 'https://DOMAIN_ENDPOINT/_dashboards/auth/login' -H "osd-xsrf: true" -H "content-type:application/json" -d '{"username":"DOMAIN_ADMIN_UNAME", "password" : "DOMAIN_ADMIN_PW"} ' -c auth.txt

# Load Default Dashboard
curl -XPOST 'https://DOMAIN_ENDPOINT/_dashboards/api/saved_objects/_import' -H "osd-xsrf:true" -b auth.txt --form file=@export_opensearch_dashboards_V1_0.ndjson


################# Index Templates and ISM ###################
# Create ISM ploicy to delete data after 366 days
curl -s -XPUT -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW "https://DOMAIN_ENDPOINT/_opendistro/_ism/policies/domains" -H 'Content-Type: application/json' -d'{"policy":{"ism_template":{"index_patterns" : ["domains-*", "cwl-*"]},"policy_id":"domains","description":"hot-delete workflow","last_updated_time":1612206385815,"schema_version":1,"error_notification":null,"default_state":"hot","states":[{"name":"hot","actions":[],"transitions":[{"state_name":"delete","conditions":{"min_index_age":"366d"}}]},{"name":"delete","actions":[{"delete":{}}],"transitions":[]}]}}'

# Create Template
curl -s -XPUT -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW "https://DOMAIN_ENDPOINT/_template/domains" -H 'Content-Type: application/json' -d'{"index_patterns":["domains-*", "cwl-*"],"settings":{"number_of_shards":1,"number_of_replicas":1}}'

################# Pipiline CREATION ###################
curl -s -XPUT -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW "https://DOMAIN_ENDPOINT/_ingest/pipeline/search-slow-logs" -H 'Content-Type: application/json' -d'{"description":"Pipeline for parsing opensearch search slow logs in plaintext format.","processors":[{"grok":{"field":"@message","pattern_definitions":{"GREEDYMULTILINE":"(.|\n)*","INDEXNAME":"[a-zA-Z0-9_.-]*"},"patterns":["\\[%{TIMESTAMP_ISO8601:opensearch.slowlog.timestamp}\\]\\[%{LOGLEVEL:opensearch.slowlog.level}\\s*\\]\\[%{DATA:opensearch.slowlog.logger}\\]\\s*\\[%{DATA:opensearch.slowlog.node}\\]\\s*\\[%{DATA:opensearch.slowlog.index.name}\\]\\s*\\[%{DATA:opensearch.slowlog.shard.id:int}]\\stook\\[%{DATA:opensearch.slowlog.took}\\],\\stook_millis\\[%{DATA:opensearch.slowlog.took_millis:float}\\],\\stotal_hits\\[%{DATA:opensearch.slowlog.total_hits:int}\\+\\shits\\]\\,\\stypes\\[%{DATA:opensearch.slowlog.types}\\],\\sstats\\[%{DATA:opensearch.slowlog.stats}\\],\\ssearch_type\\[%{DATA:opensearch.slowlog.search_type}\\],\\stotal_shards\\[%{DATA:opensearch.slowlog.total_shards:int}\\],\\ssource\\[%{GREEDYDATA:opensearch.slowlog.source_query}\\],\\sid\\[%{DATA:opensearch.slowlog.x-opaque-id}\\]","\\[%{TIMESTAMP_ISO8601:opensearch.slowlog.timestamp}\\]\\[%{LOGLEVEL:opensearch.slowlog.level}\\s*\\]\\[%{DATA:opensearch.slowlog.logger}\\]\\s*\\[%{DATA:opensearch.slowlog.node}\\]\\s*\\[%{DATA:opensearch.slowlog.index.name}\\]\\s*\\[%{DATA:opensearch.slowlog.shard.id:int}]\\stook\\[%{DATA:opensearch.slowlog.took}\\],\\stook_millis\\[%{DATA:opensearch.slowlog.took_millis:float}\\],\\stotal_hits\\[%{DATA:opensearch.slowlog.total_hits:int}\\shits\\]\\,\\stypes\\[%{DATA:opensearch.slowlog.types}\\],\\sstats\\[%{DATA:opensearch.slowlog.stats}\\],\\ssearch_type\\[%{DATA:opensearch.slowlog.search_type}\\],\\stotal_shards\\[%{DATA:opensearch.slowlog.total_shards:int}\\],\\ssource\\[%{GREEDYDATA:opensearch.slowlog.source_query}\\],\\sid\\[%{DATA:opensearch.slowlog.x-opaque-id}\\]"]}},{"date":{"field":"search.slowlog.timestamp","target_field":"@timestamp","formats":["yyyy-MM-ddTHH:mm:ss,SSS"],"if":"ctx.event.timezone == null","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"if":"ctx.event.timezone != null","field":"search.slowlog.timestamp","target_field":"@timestamp","formats":["yyyy-MM-ddTHH:mm:ss,SSS"],"timezone":"{{ event.timezone }}","on_failure":[{"append":{"value":"{{ _ingest.on_failure_message }}","field":"error.message"}}]}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}'

curl -s -XPUT -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW "https://DOMAIN_ENDPOINT/_ingest/pipeline/index-slow-logs" -H 'Content-Type: application/json' -d'{"description":"Pipeline for parsing opensearch index slow logs in plaintext format.","processors":[{"grok":{"field":"@message","pattern_definitions":{"GREEDYMULTILINE":"(.|\n)*","INDEXNAME":"[a-zA-Z0-9_.-]*"},"patterns":["\\[%{TIMESTAMP_ISO8601:opensearch.slowlog.timestamp}\\]\\[%{WORD:opensearch.slowlog.level}(%{SPACE})\\]\\[%{DATA:opensearch.slowlog.logger}\\]%{SPACE}\\[%{DATA:opensearch.slowlog.node}\\](%{SPACE})?(\\[%{INDEXNAME:opensearch.slowlog.index.name}\\]\\[%{NUMBER:opensearch.slowlog.shard.id}\\])?(%{SPACE})?(\\[%{INDEXNAME:opensearch.slowlog.index.name}\\/%{DATA:opensearch.slowlog.index.id}\\])?(%{SPACE})?%{SPACE}(took\\[%{DATA:opensearch.slowlog.took}\\],)?%{SPACE}(took_millis\\[%{NUMBER:opensearch.slowlog.took_millis:long}\\],)?%{SPACE}(type\\[%{DATA:opensearch.slowlog.type}\\],)?%{SPACE}(id\\[%{DATA:opensearch.slowlog.id}\\],)?%{SPACE}(routing\\[%{DATA:opensearch.slowlog.routing}\\],)?%{SPACE}(total_hits\\[%{NUMBER:opensearch.slowlog.total_hits:int}\\],)?%{SPACE}(types\\[%{DATA:opensearch.slowlog.types}\\],)?%{SPACE}(stats\\[%{DATA:opensearch.slowlog.stats}\\],)?%{SPACE}(search_type\\[%{DATA:opensearch.slowlog.search_type}\\],)?%{SPACE}(total_shards\\[%{NUMBER:opensearch.slowlog.total_shards:int}\\],)?%{SPACE}(source\\[%{GREEDYMULTILINE:opensearch.slowlog.source_query}\\])?,?%{SPACE}(extra_source\\[%{DATA:opensearch.slowlog.extra_source}\\])?,?"]}},{"json":{"field":"opensearch.slowlog.source_query","target_field":"opensearch.slowlog.source_query_json","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"field":"opensearch.slowlog.timestamp","target_field":"@timestamp","formats":["yyyy-MM-ddTHH:mm:ss,SSS"],"if":"ctx.event.timezone == null","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"if":"ctx.event.timezone != null","field":"opensearch.slowlog.timestamp","target_field":"@timestamp","formats":["yyyy-MM-ddTHH:mm:ss,SSS"],"timezone":"{{ event.timezone }}","on_failure":[{"append":{"value":"{{ _ingest.on_failure_message }}","field":"error.message"}}]}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}'

curl -s -XPUT -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW "https://DOMAIN_ENDPOINT/_ingest/pipeline/application-logs" -H 'Content-Type: application/json' -d'{"description":"Pipeline for parsing the opensearch server log file in plaintext format.","processors":[{"grok":{"field":"@message","pattern_definitions":{"GREEDYMULTILINE":"(.|\n)*","INDEXNAME":"[a-zA-Z0-9_.-]*","GC_ALL":"\\[gc\\]\\[%{NUMBER:opensearch.server.gc.overhead_seq}\\] overhead, spent \\[%{NUMBER:opensearch.server.gc.collection_duration.time:float}%{DATA:opensearch.server.gc.collection_duration.unit}\\] collecting in the last \\[%{NUMBER:opensearch.server.gc.observation_duration.time:float}%{DATA:opensearch.server.gc.observation_duration.unit}\\]","GC_YOUNG":"\\[gc\\]\\[young\\]\\[%{NUMBER:opensearch.server.gc.young.one}\\]\\[%{NUMBER:opensearch.server.gc.young.two}\\]%{SPACE}%{GREEDYMULTILINE:message}","LOG_HEADER":"\\[%{TIMESTAMP_ISO8601:opensearch.server.timestamp}\\]\\[%{LOGLEVEL:log.level}%{SPACE}\\]\\[%{DATA:opensearch.component}%{SPACE}\\](%{SPACE})?(\\[%{DATA:opensearch.node.name}\\])?(%{SPACE})?"},"patterns":["%{LOG_HEADER}%{GC_ALL}","%{LOG_HEADER}%{GC_YOUNG}","%{LOG_HEADER}%{SPACE}((\\[%{INDEXNAME:opensearch.index.name}\\]|\\[%{INDEXNAME:opensearch.index.name}\\/%{DATA:opensearch.index.id}\\]))?%{SPACE}%{GREEDYMULTILINE:message}"]}},{"date":{"target_field":"@timestamp","formats":["yyyy-MM-dd'T'HH:mm:ss,SSS"],"if":"ctx.event.timezone == null","field":"opensearch.server.timestamp","on_failure":[{"append":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}},{"date":{"formats":["yyyy-MM-dd'T'HH:mm:ss,SSS"],"timezone":"{{ event.timezone }}","if":"ctx.event.timezone != null","field":"opensearch.server.timestamp","target_field":"@timestamp","on_failure":[{"append":{"value":"{{ _ingest.on_failure_message }}","field":"error.message"}}]}}],"on_failure":[{"set":{"field":"error.message","value":"{{ _ingest.on_failure_message }}"}}]}'

################# ALERTS CREATION ###################

# Create Destination for E-mail alert
destination_id=`curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/destinations' -H 'Content-Type: application/json' -d'{"name":"cdk_monitoring_email","type":"sns","sns":{"role_arn":"SNS_ROLE_ARN","topic_arn":"SNS_TOPIC_ARN"}}' | jq -r '._id'`

# Create Monitor for Cluster Status Yellow and send an alert if its yellow for last 30 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"cluster_health_yellow","enabled":true,"schedule":{"period":{"interval":30,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"domain_yellow":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-30m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"term":{"ClusterStatus.yellow":{"value":1,"boost":1}}}]}}}}}],"triggers":[{"name":"cluster_health_yellow","severity":"3","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"cluster_health_yellow_alert","destination_id":"'$destination_id'","message_template":{"source":"One or more of your cluster health has been turned into YELLOW between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.domain_yellow.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.domain_yellow.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"cluster_health_yellow_alert"}}]}]}'




# Create Monitor for Cluster Status Red and send an alert if its RED every 5 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"cluster_health_red","enabled":true,"schedule":{"period":{"interval":5,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"domain_red":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-5m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"term":{"ClusterStatus.red":{"value":1,"boost":1}}}]}}}}}],"triggers":[{"name":"cluster_health_red","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"cluster_health_red_alert","destination_id":"'$destination_id'","message_template":{"source":"One or more of your cluster health has been turned into RED between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.domain_red.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.domain_red.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"cluster_health_red_alert"}}]}]}'


# Create Monitor for Cluster when ClusterIndexWritesBlocked is 1 for last 5 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"ClusterIndexWritesBlocked","enabled":true,"schedule":{"period":{"interval":5,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"ClusterIndexWritesBlocked":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-5m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"term":{"ClusterIndexWritesBlocked":{"value":1,"boost":1}}}]}}}}}],"triggers":[{"name":"ClusterIndexWritesBlocked","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"ClusterIndexWritesBlocked_Alert","destination_id":"'$destination_id'","message_template":{"source":"One or more of your ClusterIndexWritesBlocked has been turned into 1 between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.ClusterIndexWritesBlocked.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.ClusterIndexWritesBlocked.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"ClusterIndexWritesBlocked_Alert"}}]}]}'


# Create Monitor for Cluster when AutomatedSnapshotFailure is 1 for last 5 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"AutomatedSnapshotFailure","enabled":true,"schedule":{"period":{"interval":5,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"AutomatedSnapshotFailure":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-5m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"term":{"AutomatedSnapshotFailure":{"value":1,"boost":1}}}]}}}}}],"triggers":[{"name":"AutomatedSnapshotFailure","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"AutomatedSnapshotFailure_Alert","destination_id":"'$destination_id'","message_template":{"source":"One or more of your AutomatedSnapshotFailure has been turned into 1 between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.AutomatedSnapshotFailure.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.AutomatedSnapshotFailure.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"AutomatedSnapshotFailure_Alert"}}]}]}'


# Create Monitor for Cluster when KibanaHealthyNodes is 0 for last 15 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"KibanaHealthyNodes","enabled":true,"schedule":{"period":{"interval":15,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"KibanaHealthyNodes":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-15m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"term":{"KibanaHealthyNodes":{"value":0,"boost":1}}}]}}}}}],"triggers":[{"name":"KibanaHealthyNodes","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"KibanaHealthyNodes_Alert","destination_id":"'$destination_id'","message_template":{"source":"One or more of your KibanaHealthyNodes has been turned into 0 between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.KibanaHealthyNodes.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.KibanaHealthyNodes.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"KibanaHealthyNodes_Alert"}}]}]}'


# Create Monitor for Cluster when OpenSearchDashboardsHealthyNode is 0 for last 15 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"OpenSearchDashboardsHealthyNode","enabled":true,"schedule":{"period":{"interval":15,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"OpenSearchDashboardsHealthyNode":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-15m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"term":{"OpenSearchDashboardsHealthyNode":{"value":0,"boost":1}}}]}}}}}],"triggers":[{"name":"OpenSearchDashboardsHealthyNode","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"OpenSearchDashboardsHealthyNode","destination_id":"'$destination_id'","message_template":{"source":"One or more of your OpenSearchDashboardsHealthyNode has been turned into 0 between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.OpenSearchDashboardsHealthyNode.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.OpenSearchDashboardsHealthyNode.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"OpenSearchDashboardsHealthyNode_Alert"}}]}]}'



# Create Monitor for Cluster when InvalidHostHeaderRequests is 1 for last 15 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"InvalidHostHeaderRequests","enabled":true,"schedule":{"period":{"interval":15,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"InvalidHostHeaderRequests":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-15m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"term":{"InvalidHostHeaderRequests":{"value":1,"boost":1}}}]}}}}}],"triggers":[{"name":"InvalidHostHeaderRequests","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"InvalidHostHeaderRequests_Alert","destination_id":"'$destination_id'","message_template":{"source":"One or more of your InvalidHostHeaderRequests has been turned into 1 between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.InvalidHostHeaderRequests.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.InvalidHostHeaderRequests.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"InvalidHostHeaderRequests_Alert"}}]}]}'


# Create Monitor for Cluster when CPUUtilization is > 80% in last 15 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"CPUUtilization","enabled":true,"schedule":{"period":{"interval":15,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"CPUUtilization":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-15m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"range":{"CPUUtilization":{"gte":80}}}]}}}}}],"triggers":[{"name":"CPUUtilization","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"CPUUtilization_Alert","destination_id":"'$destination_id'","message_template":{"source":"CPU of one or more of your Domain has reached 80% between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.CPUUtilization.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.CPUUtilization.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"CPUUtilization_Alert"}}]}]}'


# Create Monitor for Cluster when JVMMemoryPressure is > 80% in last 5 mins
curl -s -XPOST -u DOMAIN_ADMIN_UNAME:DOMAIN_ADMIN_PW 'https://DOMAIN_ENDPOINT/_opendistro/_alerting/monitors' -H 'Content-Type: application/json' -d'{"type":"monitor","name":"JVMMemoryPressure","enabled":true,"schedule":{"period":{"interval":5,"unit":"MINUTES"}},"inputs":[{"search":{"indices":["domains-*"],"query":{"size":0,"aggs":{"JVMMemoryPressure":{"terms":{"field":"domain_name.keyword","size":25}}},"query":{"bool":{"filter":[{"range":{"@timestamp":{"from":"{{period_end}}||-5m","to":"{{period_end}}","include_lower":true,"include_upper":true,"format":"epoch_millis","boost":1}}},{"range":{"JVMMemoryPressure":{"gte":80}}}]}}}}}],"triggers":[{"name":"JVMMemoryPressure","severity":"1","condition":{"script":{"source":"ctx.results[0].hits.total.value > 0","lang":"painless"}},"actions":[{"name":"JVMMemoryPressure_Alert","destination_id":"'$destination_id'","message_template":{"source":"JVM of one or more of your Domain has reached 80% between {{ctx.periodStart}} and {{ctx.periodEnd}}, Please find below details about the domain for further actions and troubleshooting. \n\n- Severity: {{ctx.trigger.severity}}\n- Domain names: \n  {{#ctx.results.0.aggregations.JVMMemoryPressure.buckets}} \n    {{key}} https://'$InstanceIP'/_dashboards/app/dashboards#/view/19087650-454f-11eb-87ad-632020bc8bdf?_a=(query:(language:kuery,query:%27domain_name%20:%20%22{{key}}%22%27)) \n{{/ctx.results.0.aggregations.JVMMemoryPressure.buckets}}","lang":"mustache"},"throttle_enabled":false,"subject_template":{"source":"JVMMemoryPressure_Alert"}}]}]}'


echo -e "Completed Dashboards alerts and dashboard generation"