{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# Setup environment\n", "This notebook loads environment variables and saves them for later use in any other notebook." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Star GitHub repo" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%html\n", "\n", "Star\n", "" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## _Click This Button ^^ Above ^^_" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Get domain id" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import json\n", "\n", "NOTEBOOK_METADATA_FILE = \"/opt/ml/metadata/resource-metadata.json\"\n", "domain_id = None\n", "\n", "if os.path.exists(NOTEBOOK_METADATA_FILE):\n", " with open(NOTEBOOK_METADATA_FILE, \"rb\") as f:\n", " domain_id = json.loads(f.read()).get('DomainId')\n", " print(f\"SageMaker domain id: {domain_id}\")\n", "\n", "%store domain_id" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Define `get_environment` function" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import boto3\n", "\n", "sm = boto3.client(\"sagemaker\")\n", "ssm = boto3.client(\"ssm\")\n", "\n", "def get_environment(ssm_params):\n", " r = sm.describe_domain(\n", " DomainId=[\n", " d[\"DomainId\"] for d in sm.list_domains()[\"Domains\"] if boto3.Session().region_name in d[\"DomainArn\"]\n", " ][0]\n", " )\n", " del r[\"ResponseMetadata\"]\n", " del r[\"CreationTime\"]\n", " del r[\"LastModifiedTime\"]\n", " r = {**r, **r[\"DefaultUserSettings\"]}\n", " del r[\"DefaultUserSettings\"]\n", "\n", " i = {\n", " **r,\n", " **{t[\"Key\"]:t[\"Value\"] \n", " for t in sm.list_tags(ResourceArn=r[\"DomainArn\"])[\"Tags\"] \n", " if t[\"Key\"] in [\"EnvironmentName\", \"EnvironmentType\"]}\n", " }\n", "\n", " for p in ssm_params:\n", " try:\n", " i[p[\"VariableName\"]] = ssm.get_parameter(Name=f\"{i['EnvironmentName']}-{i['EnvironmentType']}-{p['ParameterName']}\")[\"Parameter\"][\"Value\"]\n", " except:\n", " i[p[\"VariableName\"]] = \"\"\n", "\n", " return i\n", "\n", "def get_session(region, default_bucket):\n", " \"\"\"Gets the sagemaker session based on the region.\n", "\n", " Args:\n", " region: the aws region to start the session\n", " default_bucket: the bucket to use for storing the artifacts\n", "\n", " Returns:\n", " sagemaker.session.Session instance\n", " \"\"\"\n", "\n", " boto_session = boto3.Session(region_name=region)\n", "\n", " sagemaker_client = boto_session.client(\"sagemaker\")\n", " runtime_client = boto_session.client(\"sagemaker-runtime\")\n", " return sagemaker.session.Session(\n", " boto_session=boto_session,\n", " sagemaker_client=sagemaker_client,\n", " sagemaker_runtime_client=runtime_client,\n", " default_bucket=default_bucket,\n", " )" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Load and output environment variables" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# Dynamically load environmental SSM parameters - provide the list of the variables to load from SSM parameter store\n", "ssm_parameters = [\n", " {\"VariableName\":\"DataBucketName\", \"ParameterName\":\"data-bucket-name\"},\n", " {\"VariableName\":\"ModelBucketName\", \"ParameterName\":\"model-bucket-name\"},\n", " {\"VariableName\":\"S3VPCEId\", \"ParameterName\":\"s3-vpce-id\"},\n", " {\"VariableName\":\"S3KmsKeyId\", \"ParameterName\":\"kms-s3-key-arn\"},\n", " {\"VariableName\":\"EbsKmsKeyArn\", \"ParameterName\":\"kms-ebs-key-arn\"},\n", " {\"VariableName\":\"PipelineExecutionRole\", \"ParameterName\":\"sm-pipeline-execution-role-arn\"},\n", " {\"VariableName\":\"ModelExecutionRole\", \"ParameterName\":\"sm-model-execution-role-name\"},\n", " {\"VariableName\":\"StackSetExecutionRole\", \"ParameterName\":\"stackset-execution-role-name\"},\n", " {\"VariableName\":\"StackSetAdministrationRole\", \"ParameterName\":\"stackset-administration-role-arn\"},\n", " {\"VariableName\":\"StagingAccountList\", \"ParameterName\":\"staging-account-list\"},\n", " {\"VariableName\":\"ProdAccountList\", \"ParameterName\":\"production-account-list\"},\n", " {\"VariableName\":\"EnvTypeStagingName\", \"ParameterName\":\"env-type-staging-name\"},\n", " {\"VariableName\":\"EnvTypeProdName\", \"ParameterName\":\"env-type-prod-name\"},\n", "]\n", "\n", "env_data = get_environment(ssm_params=ssm_parameters)\n", "print(f\"Environment data:\\n{json.dumps(env_data, indent=2)}\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Setup network configuration" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sagemaker.network import NetworkConfig\n", "\n", "# Network Isolation mode; default is off\n", "enable_network_isolation = False\n", "\n", "# Intercontainer traffic encryption is on\n", "encrypt_inter_container_traffic = True\n", "\n", "network_config = NetworkConfig(\n", " enable_network_isolation=enable_network_isolation, \n", " security_group_ids=env_data[\"SecurityGroups\"],\n", " subnets=env_data[\"SubnetIds\"],\n", " encrypt_inter_container_traffic=encrypt_inter_container_traffic)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Store environment variables" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "security_group_ids = env_data[\"SecurityGroups\"]\n", "subnets = env_data[\"SubnetIds\"]\n", "processing_role = env_data[\"ExecutionRole\"]\n", "execution_role = env_data[\"ExecutionRole\"]\n", "pipeline_role = env_data[\"PipelineExecutionRole\"]\n", "model_execution_role = env_data[\"ModelExecutionRole\"]\n", "data_bucket = env_data[\"DataBucketName\"]\n", "model_bucket = env_data[\"ModelBucketName\"]\n", "ebs_key_id = env_data[\"EbsKmsKeyArn\"]\n", "s3_key_id = env_data[\"S3KmsKeyId\"]\n", "env_name = env_data[\"EnvironmentName\"]\n", "env_type = env_data[\"EnvironmentType\"]\n", "env_type_staging_name = env_data[\"EnvTypeStagingName\"]\n", "env_type_prod_name = env_data[\"EnvTypeProdName\"]\n", "\n", "%store security_group_ids\n", "%store subnets\n", "%store processing_role\n", "%store execution_role\n", "%store model_execution_role\n", "%store pipeline_role\n", "%store data_bucket\n", "%store model_bucket\n", "%store ebs_key_id\n", "%store s3_key_id\n", "%store network_config\n", "%store env_name\n", "%store env_type\n", "%store env_type_staging_name\n", "%store env_type_prod_name" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "setup_done = True\n", "%store setup_done\n", "\n", "%store" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Release resources" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "%%html\n", "\n", "

Shutting down your kernel for this notebook to release resources.

\n", "\n", " \n", "" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (Data Science)", "language": "python", "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.10" } }, "nbformat": 4, "nbformat_minor": 4 }