B {0cO@sddlZddlZddlZddlZddlmZddlmZddlmZm Z m Z m Z m ZmZmZmZmZmZmZmZmZddlm Z ddlmZGddde Z!dS)N)datetime) Construct) SizeDurationStackaws_iamaws_s3aws_s3_deployment aws_lambdaaws_sns_subscriptionsaws_apigatewayaws_glueaws_kinesisfirehose aws_location aws_athena)KinesisFirehoseToS3cs&eZdZeeddfdd ZZS)CdkDatalakeMvpStackN)scope construct_idreturnc%, s|tj||f|dd}|}dd}dd}d}|} ||| ||} d} d } d |} d |}d }tj|| tjjtjj| d }tj||tjjtjj|d }tj |dddd}t j |dt dt j dgd}t j |dt dt j dt j dgd}t j|ddt jt jjdg|j|jdgdgd}||t j|dd t jt jjd!d"g|j|jdgdgd}||tj|d#tjjtjd$d%d&td'd(td)id*}| d+t j|d,d,t jt jjd!d-gd.| dd/gdgd}|j!"|t#j$j%d.| |j&t#j$j'd0d1d2t#j$j(d3d4d5d6d7t#j$j)d3t#j$j*t#j$j+t#j$j,d8d9d:id8d;d<d=t#j$j-t#j$j.t#j$/d>d?t#j$j0||d@|j&dAdBdCdDt#j$j1d3t#j$j2dEdFdGdHdIt#j$j3d3t#j$j4dJt#j$j5dK|j6dLgdMt#j$j4dNt#j$j5dOdPdLt#j$j5dQdRdLgdMgdSdTdU }t7|| t#j8| |dVdW|dX}t9j:|dYdZ| d[}t j |d\t d]d^}|;t j|jdbt>dcddde}dft?j@|dgdhi}tAjB|| | tAjCtAjDjEd3did3djd3dk}|jFGdl} tAjHdYdmtIjJdndodpt>||tAjKdqdrgdsdt}!| jLdp|!dudqigdvtMjN|dwdx|j&tMjNjOdydzd{d|d}| d~igi|ddddd }"tMjP|d||ddd}#tMjQ|d||tMjQjRdAdAddddxddddddddd tMjQjStMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddtMjQjTdddgd}| d~dddddidddd}$|$U|#dS)NcSstd}|}|d}|S)NstsZAccount)boto3clientZget_caller_identity)Z sts_clientresponse account_idrR/home/ec2-user/environment/apg-cdk-code/cdk_datalake_mvp/cdk_datalake_mvp_stack.pyget_account_ids z4CdkDatalakeMvpStack.__init__..get_account_idcSs|tj|<dS)N)osenviron) variable_namevaluerrrstore_variable,sz4CdkDatalakeMvpStack.__init__..store_variablecSs tj|S)N)rr)r rrrretrieve_variable0sz7CdkDatalakeMvpStack.__init__..retrieve_variablerZfirehose_s3_mvpZapigateway_to_firehosezmvp-cdk-bucket-zdl-athenaresults-Zglue_database_mvp)block_public_access encryption bucket_nameZMyCfnPlaceIndexZHereZ DeviceLoc) data_source index_nameZGlueRolezglue.amazonaws.comzservice-role/AWSGlueServiceRole) assumed_bymanaged_policiesZFirehoseExtendedConfigRolezfirehose.amazonaws.comZAWSLambda_FullAccessZFirehoseExtendedConfigPolicyZFirehosePolicyzs3:*z/*)effectactions resources) policy_name statementsZExtendedPolicyGlueZDataLakeAccess_Policyz s3:PutObjectz s3:GetObjectZ HelloHandlerzcdk_datalake_mvp/lambdazhello.lambda_handlerzheartbeat-decodeiZ Deploy_TSz%m/%d/%Y, %H:%M:%S)runtimecodehandler function_nametimeout environmentZLiveZLambdaCustomPolicyzgeo:SearchPlaceIndexForPositionz arn:aws:s3:::z8arn:aws:geo:us-east-1:112472565218:place-index/DeviceLoc<@)interval_in_seconds size_in_m_bsTz,/aws/kinesisfirehose/imodal_message_deliveryZDestinationDelivery)enabledlog_group_namelog_stream_nameZ UNCOMPRESSEDFZcolumn_to_json_key_mappings_keycolumnToJsonKeyMappings)case_insensitivecolumn_to_json_key_mappings(convert_dots_in_json_keys_to_underscores)open_x_json_ser_de) deserializer)parquet_ser_de) serializerz us-east-1stageLATEST) catalog_id database_nameregionrole_arn table_name version_id)r:input_format_configurationoutput_format_configurationschema_configurationi,)duration_in_seconds)r: retry_optionszerror/z`stage/!{partitionKeyFromQuery:year}/!{partitionKeyFromQuery:month}/!{partitionKeyFromQuery:day}/LambdaZ LambdaArn)parameter_nameparameter_value)type parametersZMetadataExtractionZMetadataExtractionQueryzc{year:.date_time| strftime("%Y"), month:.date_time| strftime("%m"), day:.date_time| strftime("%d")}ZJsonParsingEnginezJQ-1.6)r: processorsZDisabled) bucket_arnrJbuffering_hintscloud_watch_logging_optionscompression_format$data_format_conversion_configuration"dynamic_partitioning_configurationerror_output_prefixprefixprocessing_configurations3_backup_modeZ DirectPut)delivery_stream_name%extended_s3_destination_configurationdelivery_stream_type)Zkinesis_firehose_propsZexisting_bucket_objfirehoseZdeliverystream)serviceresource resource_namezapigw-to-fh-rolezapigateway.amazonaws.com)r)zfirehose:PutRecordzfirehose:PutRecordBatch)r-r,z$input.params('stream-name')z$$util.base64Encode($input.json('$')))ZData)ZDeliveryStreamNameZRecordzapplication/json)indentdev) logging_leveldata_trace_enabled stage_nametracing_enabled) rest_api_namedeploy_optionscloud_watch_rolez {stream-name}ZCDK_DEFAULT_REGIONZ PutRecordPOSTZ200) status_code)credentials_rolerequest_templatesintegration_responses)rfpathrIactionintegration_http_methodoptions statusCode)method_responses GlueCrawlerzimodaldb_stage-testzcron(0 4 * * ? *))schedule_expression s3Targetsryzs3://z/stage/ZLOG)update_behaviordelete_behaviorzM{"Version":1.0,"Grouping":{"TableGroupingPolicy":"CombineCompatibleSchemas"}})namerolescheduletargetsrHschema_change_policy configurationZ GlueDatabaseZimodaldb)r description)rGdatabase_inputZ GlueTableZEXTERNAL_TABLEz1.0parquetnone,falseZ1152fileZ933014Z321) Z crawlerSchemaDeserializerVersionZcrawlerSchemaSerializerVersionZupdated_by_crawlerclassificationcompressionType delimiterZhas_encrypted_dataZaverageRecordSizeZ typeOfDataZsizeKeyZ record_countZpacketidint)rrUZ devicetypeZdeviceidZuserapplicationidZorganizationidlenstatusZ hiverxtimestring longitudeZdoublelatitudeZ addressnumberZstreetZ municipalityrIZ subregionZ postalcodecountryZ timezone_nametimezone_offsetZsolar_panel_currentZbattery_currentZsolar_panel_voltageZbattery_voltageZ date_timeZ'no_messages_sent_since_last_power_cycleZaltitudeZspeedZcourseZlast_rssi_valueZ modem_currentz=org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormatz>org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormatz;org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDezserialization.format)serializationLibraryrV)columnslocation input_format output_format serde_info)rr table_typerVstorage_descriptor)rGrH table_input)Vsuper__init__s3BucketBlockPublicAccess BLOCK_ALLBucketEncryption S3_MANAGEDr CfnPlaceIndexiamRoleServicePrincipal ManagedPolicyfrom_aws_managed_policy_namePolicyPolicyStatementEffectALLOWrXattach_to_role_lambdaFunctionRuntime PYTHON_3_8Code from_assetrsecondsrnowstrftime add_aliasrattach_inline_policyreCfnDeliveryStream*ExtendedS3DestinationConfigurationPropertyrJBufferingHintsProperty CloudWatchLoggingOptionsProperty)DataFormatConversionConfigurationProperty InputFormatConfigurationPropertyDeserializerPropertyOpenXJsonSerDeProperty!OutputFormatConfigurationPropertySerializerPropertyParquetSerDePropertySchemaConfigurationProperty(DynamicPartitioningConfigurationPropertyRetryOptionsPropertyProcessingConfigurationPropertyProcessorPropertyProcessorParameterProperty function_arnrCfnDeliveryStreamPropsr format_arn add_to_policykinesis_firehoseattr_arndictjsondumps apigatewayRestApi StageOptionsMethodLoggingLevelINFOroot add_resourceAwsIntegrationrrIntegrationResponse add_methodglue CfnCrawlerScheduleProperty CfnDatabaseCfnTableTableInputPropertyStorageDescriptorPropertyColumnPropertyadd_depends_on)%selfrrkwargsrrr"r#r r!Z stored_valueZkinesis_delivery_stream_nameZapi_gateway_namer&Zathena_bucket_nameZglue_database_namebucketZathenaresultsbucketZcfn_place_indexZ glue_roleZfirehose_extended_config_roleZfirehose_extended_config_policyZglue_extended_policyZ my_lambdaZ lambda_customrcrZkinesis_delivery_stream_arnZ api_fh_roleZapi_to_fh_template_sectionZapi_to_fh_request_mappingapiZ fh_backendZfh_backend_integrationrZ gluedatabaseZ gluetable) __class__rrrsB               zCdkDatalakeMvpStack.__init__)__name__ __module__ __qualname__rstrr __classcell__rr)rrrsr)"rgrrrr constructsraws_cdkrrrrrrrr Zs3deployr rr subsr rr rrrerrrZathenaZ0aws_solutions_constructs.aws_kinesis_firehose_s3rrrrrrs  <