a -a:@sddlmZmZmZmZm Z m Z m ZmZmZmZmZmZmZmZmZmZm Z!m"Z#m$Z%m&Z'ddl(Z(ddlmZGdddej)Z*dS))core aws_kinesisaws_ec2aws_elasticsearchaws_kinesisanalyticsaws_iam aws_cloud9aws_ecsaws_ecs_patternsaws_rds aws_databrew aws_cognitoaws_s3aws_elasticloadbalancingv2custom_resourcesaws_logsaws_applicationautoscalingaws_s3_deployment aws_eventsaws_events_targetsN)rcs(eZdZejeddfdd ZZS)FlinkSqlDemoStackN)scope construct_idreturnc s tj||fi|t|j}t|j}tj|dddd}tj |dtj j dd}t j |d|t jd gd }tj|d d d } tj|ddd} tj|d| d} | jtjd tjddtj|dtj j tjjtjddd| tjtj j!dgtj"dtj#j$ddtj%ddd| gtj&ddt'j(dgt'j)j*t'+dgd gd!} t,j-|d"| dt,j.t/j0j1d#d$d%| j2id&dd'} | j3j4j5tj|j6tjdd(tj7|d)| tj8j9dtjdtj j!d*d+}tj7|d,| tj8j:dtjdtj j!d*d+}t'j;|d-t'j(d.gt'j)j*d/<||d0<||d1<||gd2t'j(d3gt'j)j*d4<||d1<||gd2t'j(gd5t'j)j*d6<||d/<||d1<||gd2t'j(d7gt'j)j*d1<||gd2gd8}t'j=|d9t'>d:|gd;}| ?||j@t'j;Ad<d=|B|tCjD|d>d?d@|jEdA}tj|dB| dCdD}gdE}|D]\}tFjG|dF|tFjGjHtFjGjIdGdH|dIdJdKdLtFjGjJtFjGjKdMddNdOdP|dQqtj |dRtj j dd}tFjL|dSdTtFjLjMtFjLjNdUdVdWdXdYdZd[d\gd]}t'j=|d^t'>d_d`t'jOt'j(t'j)j*dadbgdcddgdegd8idf}|B|tPjQ|dgtPjRjSdh|di|djgdktTjUjVtPjWtPjXdldgdmdn|jYidotPjWdpdm|jYdqdrdsdt}tFjZ|du|jEdP|dvdP|dvdwtFjZj[tFjZjI|j\dxdLdydtFjZj]tFjZj^dMdzdOd{gtFjZj_|j`dqd|d}d~}tPjQ|dtPjRjSdh|di|dgdktTjUjVtPjWtPjXdddmdn|jYidod}|jab||jab|t/jc|dt/jdjeddd}|jfdt/j0j1dd$|j\| jgdddt/jhjiddd| j|jk|?|jktljm|ddtljndgdg|jYgdgddtojp| jq|ddgd}tjr|dd| j3jsdd}dS)NZsourceIPStringzSource CIDR for Load Balancer)type descriptionZkdaConnectorsBucketT)removal_policyZauto_delete_objectsZflinkConnectorDeploymentz ../connectors)Zdestination_bucketZsourcesZ streamingVPCz 10.0.0.0/16)cidrZ dataStream)Z shard_countZesSG)vpcP)ZpeerZ connectionZ esClusterzt3.medium.elasticsearch)Z data_nodesZdata_node_instance_type) subnet_typed)Z volume_sizeZ volume_type)enabledF)Zavailability_zone_countzes:**)actionseffectZ principals) rversionZcapacityr Z vpc_subnetsZebsZnode_to_node_encryptionZencryption_at_restZ enforce_httpsZsecurity_groupsZzone_awarenessZaccess_policiesZesProxyServicez../elasticsearch_proxy/) directoryZ ES_ENDPOINT)image environment)r listener_portZtask_image_optionsZpublic_load_balancer)Z port_rangeZglueVPCe)Z one_per_azr")r serviceopensubnetsZ kinesisVPCeZ kdaPolicyzglue:GetDatabasez#arn:aws:glue:{}:{}:database/defaultz arn:aws:glue:{}:{}:database/hivezarn:aws:glue:{}:{}:catalog)r&r' resourceszglue:GetConnectionzarn:aws:glue:{}:{}:connection/*)z glue:GetTablezglue:GetTableszglue:CreateTablezglue:UpdateTablez arn:aws:glue:{}:{}:table/defaultzglue:GetDatabases)Z statementsZkdaRolezkinesisanalytics.amazonaws.com) assumed_byZmanaged_policiesZAmazonVPCFullAccess)policyZ kdaStudiozZEPPELIN-FLINK-1_0Z INTERACTIVE)Zruntime_environmentZapplication_modeZservice_execution_roleZkdaSGzSecurity Group for KDA Studio)r r)ZyellowZfhvZfhvhvZgreenZ nycDataSetznyc-tlcz trip data/z_tripdata_2019-05.csv)bucketkey)Zs3_input_definitionZCSV,) delimiterZ header_row)Zcsvztlc-trip-record-data-2019-may-)inputformatformat_optionsnameZ dataBucketZnycDataSetRecipez$tlc-trip-record-data-2019-may-recipeZUNIONa)[["PULocationID","DOLocationID","tpep_pickup_datetime","tpep_dropoff_datetime"],["PULocationID","DOLocationID","pickup_datetime","dropoff_datetime"],["PULocationID","DOLocationID","lpep_pickup_datetime","lpep_dropoff_datetime"],["PULocationID","DOLocationID","pickup_datetime","dropoff_datetime"]]zq["tlc-trip-record-data-2019-may-fhv","tlc-trip-record-data-2019-may-green","tlc-trip-record-data-2019-may-fhvhv"]a[{"S3InputDefinition":{"Bucket":"nyc-tlc","Key":"trip data/fhv_tripdata_2019-05.csv"}},{"S3InputDefinition":{"Bucket":"nyc-tlc","Key":"trip data/green_tripdata_2019-05.csv"}},{"S3InputDefinition":{"Bucket":"nyc-tlc","Key":"trip data/fhvhv_tripdata_2019-05.csv"}}]zN["PULocationID","DOLocationID","tpep_pickup_datetime","tpep_dropoff_datetime"])ZdatasetsColumnsZsecondaryDatasetNamesZsecondaryInputsZtargetColumnNames)Z operation parameters)action)r:Zstepsz"tlc-trip-record-data-2019-may-rolezdatabrew.amazonaws.comZ getDataPolicyz s3:ListBucketz s3:GetObjectzarn:aws:s3:::nyc-tlc/*zarn:aws:s3:::nyc-tlc)r'r&r0)r1Zinline_policiesZ publishRecipezarn:aws:databrew::z :recipe/*)r0ZCustomResourcePublishRecipeZDataBrewName)Zphysical_resource_idr<r-r;ZdeleteRecipeVersionz1.0)r>Z RecipeVersion)r<r-r;)r2 log_retention on_updateZ on_deleteZ nycDataSetJobrz-job)r3ZBZIP2)r6)locationr8Zcompression_formatZ overwriter9)r:r(ZRECIPE)role_arnZ dataset_namer:ZoutputsZreciperZ startBrewJobz:job/*ZCustomResourceStartBrewJobZ startJobRun)r2r?r@Z producerTaskZ4096Z8192)Z compatibilitycpuZ memory_mibZproducerContainerz ../producer/)Z BUCKET_NAMEZ STREAM_NAMEii )Z stream_prefix)r*r+rCZmemory_limit_mibZloggingZbrewJobeCompletedz aws.databrewzDataBrew Job State ChangeZ SUCCEEDED)ZjobNamestate)sourceZ detail_typeZdetail)clusterZtask_definitionZ task_count)r$Z event_patterntargetsZ kibanaUrlzhttp://z/_plugin/kibana)value)tsuper__init__rofregionaccountr CfnParameters3ZBucket RemovalPolicyDESTROY s3_deploymentZBucketDeploymentZSourceassetec2ZVpckinesisZStreamZ SecurityGroupZadd_ingress_ruleZPeerZipv4ZPortZtcpesZDomainZElasticsearchVersionZV7_10ZCapacityConfigZSubnetSelectionZ SubnetTypePUBLICZ EbsOptionsZEbsDeviceVolumeTypeZGP2ZEncryptionAtRestOptionsZZoneAwarenessConfigiamZPolicyStatementZEffectZALLOWZ ArnPrincipal ecs_patternsZ%ApplicationLoadBalancedFargateServiceZ'ApplicationLoadBalancedTaskImageOptionsecsZContainerImage from_assetZdomain_endpointZ load_balancerZ connectionsZ allow_fromvalue_as_stringZInterfaceVpcEndpointZInterfaceVpcEndpointAwsServiceZGLUEZKINESIS_STREAMSZ ManagedPolicyr8ZRoleZServicePrincipalZ grant_readZadd_managed_policyZfrom_aws_managed_policy_nameZgrant_read_writekdaZCfnApplicationV2rBdatabrewZ CfnDatasetZ InputPropertyZS3LocationPropertyZFormatOptionsPropertyZCsvOptionsPropertyZ CfnRecipeZRecipeStepPropertyZActionPropertyZPolicyDocumentrZAwsCustomResourceZAwsCustomResourcePolicyZfrom_sdk_callslogsZ RetentionDaysZONE_DAYZ AwsSdkCallZPhysicalResourceIdr:ZCfnJobZOutputProperty bucket_nameZOutputFormatOptionsPropertyZCsvOutputOptionsPropertyZRecipePropertyrefnodeadd_dependencyZTaskDefinitionZ CompatibilityZFARGATEZ add_containerZ stream_nameZ LogDriverrZ grant_writeZ task_roleeventsZRuleZ EventPattern events_targetZEcsTaskrG CfnOutputload_balancer_dns_name)selfrrkwargsrM account_idrEZconnectors_bucketZdeploy_connectorr Z data_streamZ es_cluster_sgZ es_clusterZes_proxy_serviceZ glue_vpceZ kinesis_vpceZ kda_policyZkda_roleZ kda_studioZkda_sgZ data_sourcessrcZ data_bucketZ brew_recipeZ brew_roleZpublish_recipeZbrew_jobZstart_brew_jobZproducer_task_definitionZbrew_job_completed_ruleZ kibana_url __class__X/Users/flomair/PycharmProjects/flink-sql-demo/cdk/flink_sql_demo/flink_sql_demo_stack.pyrK"sv      "               '               zFlinkSqlDemoStack.__init__)__name__ __module__ __qualname__cdk ConstructstrrK __classcell__rorormrpr sr)+aws_cdkrrtrrVrrUrrWrr^rrYrZcloud9r r[r rZr Zrdsr r_r ZcognitorrPrZelbrrr`rZ autoscalingrrSrrerrfdatetimeStackrrorororpsX