import boto3 import time import os import logging import re ''' Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at http://aws.amazon.com/apache2.0/ or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. Modified by Suman Koduri (skkoduri@amazon.com) ''' from datetime import datetime LOGLEVEL = os.getenv('LOG_LEVEL', 'ERROR').strip() logger = logging.getLogger() logger.setLevel(LOGLEVEL) class RDSBackupToolException(Exception): pass def search_tag_created(response): # Takes a describe_db_cluster_snapshots response and searches for our shareAndCopy tag try: for tag in response['TagList']: if tag['Key'] == 'CreatedBy' and tag['Value' ] == 'RDS Backup Tool': return True except Exception: return False else: return False def filter_rds(rds_instances, client): # Filter RDS and Aurora instances rds = paginate_api_call(client, 'describe_db_instances', 'DBInstances')['DBInstances'] aurora = paginate_api_call(client, 'describe_db_clusters', 'DBClusters')['DBClusters'] # Declare lists filtered_rds = [] # RDS Instances to back up filtered_aurora = [] # Aurora Instances to back up all_rds = [] # All RDS instances in the account all_aurora = [] # All Aurora instances in the account # Gather list of all RDS Instances for r in rds: if r['Engine'] != 'aurora': all_rds.append(r['DBInstanceIdentifier']) # Gather list of all Aurora instances for a in aurora: if a['Engine'] == 'aurora': all_aurora.append(a['DBClusterIdentifier']) # Filter RDS Instances and Aurora clusters for r in rds_instances: if any(r in s for s in all_aurora): filtered_aurora.append("{}-cluster".format(r)) elif r in all_rds: filtered_rds.append(r) else: print( "Invalid or non-existent RDS Instance/Cluster Name: {}". format(r) ) return [filtered_rds, filtered_aurora] def paginate_api_call(client, api_call, objecttype, *args, **kwargs): #Takes an RDS boto client and paginates through api_call calls and returns a list of objects of objecttype response = {} kwargs_string = ','.join( ['{}={}'.format(arg, value) for arg, value in kwargs.items()] ) if kwargs: temp_response = eval('client.{}({})'.format(api_call, kwargs_string)) else: temp_response = eval('client.{}()'.format(api_call)) response[objecttype] = temp_response[objecttype][:] while 'Marker' in temp_response: if kwargs: temp_response = eval( 'client.{}(Marker="{}",{})'.format( api_call, temp_response['Marker'], kwargs_string ) ) else: temp_response = eval( 'client.{}(Marker="{}")'.format( api_call, temp_response['Marker'] ) ) for obj in temp_response[objecttype]: response[objecttype].append(obj) return response def get_own_snapshots_source_aurora(filtered_clusters, response): # Filters our own snapshots filtered = {} for snapshot in response['DBClusterSnapshots']: client = boto3.client('rds') response_tags = client.list_tags_for_resource( ResourceName = snapshot['DBClusterSnapshotArn'] ) if snapshot['SnapshotType'] == 'manual' and snapshot[ 'DBClusterSnapshotIdentifier' ].split('-snapshot-')[0] in filtered_clusters: client = boto3.client('rds') response_tags = client.list_tags_for_resource( ResourceName = snapshot['DBClusterSnapshotArn'] ) if search_tag_created(response_tags): filtered[snapshot['DBClusterSnapshotIdentifier']] = { 'Arn': snapshot['DBClusterSnapshotArn'], 'Status': snapshot['Status'], 'DBClusterIdentifier': snapshot['DBClusterIdentifier'] } return filtered def get_own_snapshots_source_rds(filtered_instances, response): # Filters our own snapshots filtered = {} for snapshot in response['DBSnapshots']: client = boto3.client('rds') response_tags = client.list_tags_for_resource( ResourceName = snapshot['DBSnapshotArn'] ) if snapshot['SnapshotType'] == 'manual' and snapshot[ 'DBSnapshotIdentifier' ].split('-snapshot-')[0] in filtered_instances: client = boto3.client('rds') response_tags = client.list_tags_for_resource( ResourceName = snapshot['DBSnapshotArn'] ) if search_tag_created(response_tags): filtered[snapshot['DBSnapshotIdentifier']] = { 'Arn': snapshot['DBSnapshotArn'], 'Status': snapshot['Status'], 'DBInstanceIdentifier': snapshot['DBInstanceIdentifier'] } return filtered def requires_backup_aurora(backup_interval, cluster, filtered_snapshots): # Returns True if latest snapshot is older than INTERVAL latest = get_latest_snapshot_ts_aurora(cluster, filtered_snapshots) if latest is not None: backup_age = datetime.now() - latest if backup_age.total_seconds() >= (backup_interval * 60 * 60): return True else: return False elif latest is None: return True def requires_backup_rds(backup_interval, instance, filtered_snapshots): # Returns True if latest snapshot is older than INTERVAL latest = get_latest_snapshot_ts_rds(instance, filtered_snapshots) if latest is not None: backup_age = datetime.now() - latest if backup_age.total_seconds() >= (backup_interval * 60 * 60): return True else: return False elif latest is None: return True def get_timestamp_no_minute_rds(snapshot_identifier, snapshot_list): # Get a timestamp from the name of a snapshot and strip out the minutes pattern = '{}-snapshot-(.+)-\d+'.format( snapshot_list[snapshot_identifier]['DBInstanceIdentifier'] ) timestamp_format = '%Y-%m-%d-%H' date_time = re.search(pattern, snapshot_identifier) if date_time is not None: return datetime.strptime(date_time.group(1), timestamp_format) def get_timestamp_no_minute_aurora(snapshot_identifier, snapshot_list): # Get a timestamp from the name of a snapshot and strip out the minutes pattern = '{}-snapshot-(.+)-\d+'.format( snapshot_list[snapshot_identifier]['DBClusterIdentifier'] ) timestamp_format = '%Y-%m-%d-%H' date_time = re.search(pattern, snapshot_identifier) if date_time is not None: return datetime.strptime(date_time.group(1), timestamp_format) def get_latest_snapshot_ts_rds(instance_identifier, filtered_snapshots): # Get latest snapshot for a specific DBInstanceIdentifier timestamps = [] for snapshot, snapshot_object in filtered_snapshots.items(): if snapshot_object['DBInstanceIdentifier'] == instance_identifier: timestamp = get_timestamp_no_minute_rds( snapshot, filtered_snapshots ) if timestamp is not None: timestamps.append(timestamp) if len(timestamps) > 0: return max(timestamps) else: return None def get_latest_snapshot_ts_aurora(cluster_identifier, filtered_snapshots): # Get latest snapshot for a specific DBClusterIdentifier timestamps = [] for snapshot, snapshot_object in filtered_snapshots.items(): if snapshot_object['DBClusterIdentifier'] == cluster_identifier: timestamp = get_timestamp_no_minute_aurora( snapshot, filtered_snapshots ) if timestamp is not None: timestamps.append(timestamp) if len(timestamps) > 0: return max(timestamps) else: return None def manageSnapshotRecord(**kwargs): """Function to add/update records to/in the RDS_Snapshot_Logging DynamoDB Table.""" # Get current time stamp in "mm-dd-yyyy HH:MM" format timestamp = time.strftime("%m-%d-%Y %H:%M") action = '' snapshot_id = '' rds_name = '' snapshot_state = 'creating' emailed_bool = False deleted_bool = False # Empty dict for building the ExpressionAttributeValues for update_item below expression = {} # String for building the UpdateExpression for update_item below with default timestamp value update = 'set Created_Snapshot_Timestamp = if_not_exists(Created_Snapshot_Timestamp, :Snapshot_Timestamp), \ Updated_Snapshot_Timestamp = :Snapshot_Timestamp' expression[':Snapshot_Timestamp'] = {"S": timestamp} for key, value in kwargs.items(): if 'action' in key: action = value if 'snapshot_id' in key: snapshot_id = value if 'rds_name' in key: rds_name = value # If snapshot_region is specified, use it if 'snapshot_region' in key: expression[':Snapshot_Region'] = {"S": value} update += ',Snapshot_Region = if_no_exists( Snapshot_Region, :Snapshot_Region)' # If snapshot_state is specified, use it if 'snapshot_state' in key: expression[':Snapshot_State'] = {"S": value} update += ',Snapshot_State = :Snapshot_State' if 'rds_engine' in key: expression[':RDS_Engine'] = {"S": value} update += ',RDS_Engine = if_not_exists(RDS_Engine, :RDS_Engine)' # If emailed_bool is specified, use it if 'emailed_bool' in key: expression[':emailed'] = {"BOOL": value} update += ',Emailed = :emailed' # If deleted_bool is specified, use it if 'deleted_bool' in key: expression[':deleted'] = {"BOOL": value} update += ',Deleted = :deleted' # Action = add or update if action == '': logger.error("Action is empty. Valid values: 'add' or 'update'") return "ERROR: Action required" # Add default states if action is add if action == 'add': expression[':deleted'] = {"BOOL": deleted_bool} update += ',Deleted = :deleted' expression[':emailed'] = {"BOOL": emailed_bool} update += ',Emailed = :emailed' expression[':Snapshot_State'] = {"S": snapshot_state} update += ',Snapshot_State = :Snapshot_State' if snapshot_id == '' or rds_name == '': logger.error("Snapshot ID and RDS Identifier cannot be empty.") return False logger.debug("Update expression 'update' variable: {}".format(update)) logger.debug( "Expression Attribute Values 'expression': {}".format(expression) ) response = boto3.client("dynamodb").update_item( TableName = "RDS_Snapshot_Logging", Key = { "Snapshot_ID": { "S": snapshot_id }, "RDS_Name": { "S": rds_name } }, ReturnValues = "UPDATED_NEW", UpdateExpression = update, ExpressionAttributeValues = expression ) logger.debug("manageSnapshotRecord response: {}".format(response)) return response