import json import logging import urllib3 import threading import boto3 import time import random import os localtime = time.asctime( time.localtime(time.time()) ).replace(" "," ").replace(" ","-").replace(":","-") http = urllib3.PoolManager() appr_client = boto3.client('servicecatalog-appregistry') s3client = boto3.client('s3') stsclient = boto3.client('sts') # region = os.environ['AWS_REGION'] AccountId = stsclient.get_caller_identity()['Account'] Actlen = len(AccountId) - 4 LastFourAccountId = AccountId[Actlen:len(AccountId)] Accountregion = f"{region}-{LastFourAccountId}" logger = logging.getLogger() def loadneptune(event): headers = {'Content-Type': 'application/json'} try: for e_vnt in event['curlcmd']: print(e_vnt['endpoint']) response = http.request('POST',e_vnt['endpoint'],data=e_vnt['data'],headers=headers) logger.debug('Status code: ' + response.reason) print('Status code: ' + response.reason) except Exception as e: logger.error('cfnresponse(..) failed executing requests.put(..): ' + str(e)) def cfnresponse(event, context, responseStatus, responseData, physicalResourceId=None, noEcho=False): responseBody = {} responseBody['Status'] = responseStatus responseBody['Reason'] = 'See the details in CloudWatch Log Stream: ' + context.log_stream_name responseBody['PhysicalResourceId'] = physicalResourceId or context.log_stream_name responseBody['StackId'] = event['StackId'] responseBody['RequestId'] = event['RequestId'] responseBody['LogicalResourceId'] = event['LogicalResourceId'] responseBody['NoEcho'] = noEcho responseBody['Data'] = responseData json_responseBody = json.dumps(responseBody) headers = {'content-type' : '','content-length' : str(len(json_responseBody))} try: response = http.request('PUT',event['ResponseURL'],body=json_responseBody.encode('utf-8'),headers=headers) logger.debug('Status code: ' + response.reason) except Exception as e: logger.error('cfnresponse(..) failed executing requests.put(..): ' + str(e)) def timeout(event, context): logging.error('Execution is about to time out, sending failure response to CloudFormation') cfnresponse(event, context, 'FAILED',{}) def gen_surl(bucketname, keyname): url = s3client.generate_presigned_url(ClientMethod='get_object', Params={ 'Bucket': bucketname, 'Key': keyname}) return url def b_putpriv(bucket, key, ContentType,body): srep = s3client.put_object( ACL='private', Body=body, Bucket=bucket, Key=key, ContentType=ContentType,) #print(srep) return srep def b_get_obj(bucket, bkey): object = s3client.get_object(Bucket=bucket, Key=bkey) return(object['Body'].read().decode('utf-8')) def lgetattgrp(event,attgrp): rep = appr_client.get_attribute_group(attributeGroup=attgrp) jatt={} try: jatt = json.dumps(rep['attributes']) #print(jatt) except Exception as e: h=1 #print(rep['attributes']) out = {'id':attgrp,'name':rep['name'],'description':rep['description'],'attributes':rep['attributes']} return out def lassociatedresources(event,applicationid): rep = appr_client.list_associated_resources(application=applicationid) return rep['resources'] def lassocappgroup(event,applicationid): rep = appr_client.list_associated_attribute_groups(application=applicationid) return rep['attributeGroups'] def lapplications(event): res = appr_client.list_applications() apps=[] for app in res['applications']: attgrp=lassocappgroup(event,app['id']) apresources = lassociatedresources(event,app['id']) ag=[] resources=[] for ap in attgrp: ag.append(lgetattgrp(event,ap)) name = f"{app['name']}-{Accountregion}" apps.append({'id':app['id'],'name':name,'description':app['description'],'attributeGroups':ag,'resources':apresources}) return {'applications':apps} def l_mkneptunecsv(event,apregjson): o_app='"~id",name,description,~label\n' o_appgroup ='"~id",name,description,~label\n' o_attb ='"~id",name,~label\n' o_c_attb ='"~id",~label,~from,~to\n' o_cbine = '"~id",~label,~from,~to\n' NeptuneClusterRole = event['ResourceProperties']['iamRoleArn'] bucket = event['ResourceProperties']['bucket'] prefix = event['ResourceProperties']['Prefix'] appId=1 atb_count=1 Accountregionk = Accountregion.replace('-','') Atgrpid= len(apregjson['applications']) +1 for app in apregjson['applications']: o_app += f"{app['id']},{app['name']},{app['description']},Application\n" for attribgroup in app['attributeGroups']: o_appgroup += f"{attribgroup['id']},{attribgroup['name']},{attribgroup['description']},Attributegroups\n" o_cbine += f"App{appId}Attg{Atgrpid},Attgroup,{app['id']},{attribgroup['id']}\n" if 'attributes' in attribgroup: ajson = attribgroup['attributes'].replace('AWS::','').replace('{','').replace('}','').replace('[','').replace(']','').replace('"','').split(',') for _at in ajson: _at = _at.replace('\n','') _at = _at.replace(' ','') atbkey = f'{Accountregionk}{atb_count}' o_attb += f"{atbkey},{_at},Attributes\n" o_c_attb += f"{atbkey},Jattributes,{attribgroup['id']},{atbkey}\n" atb_count = atb_count +1 Atgrpid = Atgrpid+1 appId = appId+1 #print(o_attb) Applicationkey = f'{prefix}Application.csv' Attgroupkey = f'{prefix}Attgroup.csv' ApplicationAttgroupappskey = f'{prefix}Application_Attgroup_Application_edges.csv' Attribute2attributekey = f'{prefix}Attributegroup_attribute_Attributegroup_edges.csv' appreg2neptuneinfokey = f'{prefix}appregenv/appreg2neptuneinfokey.json' _dataloadkey = f'{prefix}outloadme.sh' _justfile = _dataloadkey.split('/')[2] Attbkey = f'{prefix}Attributes.csv' b_putpriv(bucket, Applicationkey, 'text/text',o_app) b_putpriv(bucket, Attgroupkey, 'text/text',o_appgroup) b_putpriv(bucket, ApplicationAttgroupappskey, 'text/text',o_cbine) b_putpriv(bucket, Attbkey, 'text/text',o_attb) b_putpriv(bucket, Attribute2attributekey, 'text/text',o_c_attb) mes = f'{Applicationkey},{Attgroupkey},{ApplicationAttgroupappskey} created in {bucket}' neplkey= f'{prefix}neptune_load.sh' region = os.environ['AWS_REGION'] #_nepload = b_get_obj(bucket,neplkey) _curldata = [] j_Attbkeyload = {'endpoint':'https://'+event['ResourceProperties']['endpoint']+':8182/loader','data':{'source' : 's3://'+bucket+'/'+Attbkey, 'iamRoleArn' : NeptuneClusterRole, 'format' : 'csv', 'region' : region, 'failOnError' : 'FALSE', 'parallelism' : 'MEDIUM', 'updateSingleCardinalityProperties' : 'FALSE', 'queueRequest' : 'TRUE'}} _curldata.append(j_Attbkeyload) j_appload = {'endpoint':'https://'+event['ResourceProperties']['endpoint']+':8182/loader', 'data':{'source' : 's3://'+bucket+'/'+Applicationkey, 'iamRoleArn' : NeptuneClusterRole, 'format' : 'csv', 'region' : region, 'failOnError' : 'FALSE', 'parallelism' : 'MEDIUM', 'updateSingleCardinalityProperties' : 'FALSE', 'queueRequest' : 'TRUE'}} _curldata.append(j_appload) j_Attgroupload = {'endpoint':'https://'+event['ResourceProperties']['endpoint']+':8182/loader', 'data':{'source' : 's3://'+bucket+'/'+Attgroupkey, 'iamRoleArn' : NeptuneClusterRole, 'format' : 'csv', 'region' : region, 'failOnError' : 'FALSE', 'parallelism' : 'MEDIUM', 'updateSingleCardinalityProperties' : 'FALSE', 'queueRequest' : 'TRUE'}} _curldata.append(j_Attgroupload ) j_ApplicationAttgrload = {'endpoint':'https://'+event['ResourceProperties']['endpoint']+':8182/loader', 'data':{'source' : 's3://'+bucket+'/'+ApplicationAttgroupappskey, 'iamRoleArn' : NeptuneClusterRole, 'format' : 'csv', 'region' : region, 'failOnError' : 'FALSE', 'parallelism' : 'MEDIUM', 'updateSingleCardinalityProperties' : 'FALSE', 'queueRequest' : 'TRUE'}} _curldata.append(j_ApplicationAttgrload ) j_Attgrattrbload = {'endpoint':'https://'+event['ResourceProperties']['endpoint']+':8182/loader', 'data':{'source' : 's3://'+bucket+'/'+Attribute2attributekey, 'iamRoleArn' : NeptuneClusterRole, 'format' : 'csv', 'region' : region, 'failOnError' : 'FALSE', 'parallelism' : 'MEDIUM', 'updateSingleCardinalityProperties' : 'FALSE', 'queueRequest' : 'TRUE'}} _curldata.append(j_Attgrattrbload ) # _nepfiles = [] _nepfile_Attbkey = {'region':region,'AccountId':AccountId,'bucket':bucket,'key':Attbkey,'signedurl':gen_surl(bucket,Attbkey)} _nepfile_Applicationkey = {'region':region,'AccountId':AccountId,'bucket':bucket,'key':Applicationkey,'signedurl':gen_surl(bucket,Applicationkey)} _nepfile_Attgroupkey = {'region':region,'AccountId':AccountId,'bucket':bucket,'key':Attgroupkey,'signedurl':gen_surl(bucket,Attgroupkey)} _nepfile_ApplicationAttgroupappskey = {'region':region,'AccountId':AccountId,'bucket':bucket,'key':ApplicationAttgroupappskey,'signedurl':gen_surl(bucket,ApplicationAttgroupappskey)} _nepfile_Attribute2attributekey = {'region':region,'AccountId':AccountId,'bucket':bucket,'key':Attribute2attributekey,'signedurl':gen_surl(bucket,Attribute2attributekey)} # _nepfiles.append(_nepfile_Attbkey) _nepfiles.append(_nepfile_Applicationkey) _nepfiles.append(_nepfile_Attgroupkey) _nepfiles.append(_nepfile_ApplicationAttgroupappskey) _nepfiles.append(_nepfile_Attribute2attributekey) #b_putpriv(bucket, _dataloadkey, 'text/text',_neploadsh) #print(' ') ##print(_loads3cp) mes = 'Creating CSV files for neptune_load..' appreg2neptuneinfo= {'mes':mes, 'curlcmd':_curldata,'appreg2neptuneinfokey':appreg2neptuneinfokey,'neptunefiles':_nepfiles} b_putpriv(bucket, appreg2neptuneinfokey, 'text/json',json.dumps(appreg2neptuneinfo,indent=4)) return appreg2neptuneinfo def nloadfiles(event): _ret = {} _headers = {"X-Amzn-Trace-Id":"Root=1-60f1b43d-6270e7b88f6573db5267aaa3;Sampled=0","Content-Type":"application/json"} _ep= event['ep'] print(_ep) _data = json.dumps(event['data'] ) _res = http.request('POST',_ep,headers=_headers,body=_data) _ret = json.loads( _res.data.decode('utf-8') ) return _ret def make_spoke_template(event): bucket = event['ResourceProperties']['bucket'] prefix = event['ResourceProperties']['Prefix'] template_spoke_key = f"{prefix}scappregistry_setup_base_spoke.json" template_spoke = json.loads(b_get_obj(bucket,template_spoke_key)) sc_product_spoke_key = f"{prefix}scappregistry_product.json" sc_product_spoke_key_new = f"{prefix}scappregistry_product_new.json" sc_product_template = json.loads(b_get_obj(bucket,sc_product_spoke_key)) sc_product_template ['Resources']['Lapreg2neptuneRun']['Properties']['apigwload']= event['ResourceProperties']['apigwload'] sc_product_template ['Resources']['Lapreg2neptuneRun']['Properties']['endpoint']= event['ResourceProperties']['endpoint'] sc_product_template ['Resources']['Lapreg2neptuneRun']['Properties']['iamRoleArn']= event['ResourceProperties']['iamRoleArn'] sc_product_template ['Resources']['Lapreg2neptuneRun']['Properties']['WebInterface']= f"https://{event['ResourceProperties']['bucket']}.s3.amazonaws.com/content/appreg/report/index.html" b_putpriv(bucket,sc_product_spoke_key_new,'text/json',json.dumps(sc_product_template,indent=4)) presign_sc_product_spoke_key_new = gen_surl(bucket,sc_product_spoke_key_new) template_spoke['Description'] = template_spoke['Description'] +' SpokeAccounts' template_spoke['Resources']['Productappreg']['Properties']['ProvisioningArtifactParameters'][0]['Info']['LoadTemplateFromURL']= presign_sc_product_spoke_key_new oldscenusertole = template_spoke['Parameters']['SCenduserRole'] #template_spoke ['Parameters']={} template_spoke ['Parameters']['SCenduserRole'] = oldscenusertole template_spoke ['Parameters']['SCenduserRole']['Default'] = 'user/kw_sc_enduser' template_spoke_share_key = f"{prefix}scappregistry_setup_base_spoke_share.json" b_putpriv(bucket,template_spoke_share_key,'text/json',json.dumps(template_spoke,indent=4)) presign_template_spoke_share_key = gen_surl(bucket,template_spoke_share_key) return presign_template_spoke_share_key def l_stack_res(stackid): client = boto3.client('cloudformation') s_dec = client.list_stack_resources(StackName=stackid) buf=[] for l in s_dec['StackResourceSummaries']: #buf = buf + "%s-%s-%s\n" % (l['LogicalResourceId'],l['ResourceType'],l['PhysicalResourceId']) if l['PhysicalResourceId'].find('$LATEST') < 0: buf.append(l['ResourceType']) buf = list(dict.fromkeys(buf)) return buf def chk_for_resource_attgroup(apregjson): _buf='' _ret={} eapp = {} for app in apregjson['applications']: #chk for resource #return app for _resource in app['resources']: #found a resource check for attrbgroup _found = 0 for attgrp in app['attributeGroups']: _stakattgrp = f"STACK_{_resource['name']}" if attgrp['name'] == _stakattgrp: _found =1 _buf += f"{_resource['name']} has {_found} attributeGroup\n" if _found == 0: _ret=l_stack_res(_resource['name']) if len(_ret) > 0 : eapp = {'application':app['id'],'name':_stakattgrp,'attributegroup':_stakattgrp,'description':_stakattgrp+' Description','attributes':_ret} cr_attributegroup(eapp) asso_attributegroup(eapp) print(_buf) return eapp def cr_attributegroup(eapp): rand = random.randint(130, 2400) attributes = json.dumps(eapp['attributes']) clientToken = f'ctok{rand}' response = appr_client.create_attribute_group(name=eapp['name'],description=eapp['description'], attributes=attributes,clientToken=clientToken ) def asso_attributegroup(eapp): rand = random.randint(130, 2400) response = appr_client.associate_attribute_group(application=eapp['application'],attributeGroup=eapp['attributegroup']) return response def lambda_handler(event, context): #apregjson = lapplications(event) #return chk_for_resource_attgroup(apregjson) print(json.dumps(event)) _ret={} #return lapplications(event) _ret['Message']='Creating CSV files for neptune_load..' _ret['spoketemplate']='' _ret['message'] = 'Creating CSV files for neptune_load..' _ret['WebInterface'] = f"https://{event['ResourceProperties']['bucket']}.s3.amazonaws.com/content/appreg/report/index.html" timer = threading.Timer((context.get_remaining_time_in_millis() / 1000.00) - 0.5, timeout, args=[event, context]) timer.start() status = 'SUCCESS' try: if event['RequestType'] == 'Create': apregjson = lapplications(event) chk_for_resource_attgroup(apregjson) time.sleep(4) apregjson = lapplications(event) _rret = l_mkneptunecsv(event,apregjson) _ret['Message']=_rret['mes'] #_ret['neptunefiles'] = _rret['neptunefiles'] #_ret['curl'] = _rret['curlcmd'] _levent = {'ep':event['ResourceProperties']['apigwload'],'data':_rret['neptunefiles'] } _ret['lresults'] = nloadfiles(_levent) _ret['spoketemplate'] = make_spoke_template(event) except Exception as e: logging.error('Exception: %s' % e, exc_info=True) status = 'FAILED' finally: timer.cancel() print(json.dumps(_ret)) cfnresponse(event, context, status,_ret) return _ret