{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# [모듈 2-2] SageMaker Pipelines 사용하기\n", "이 노트북에서는 아래와 같은 작업을 수행합니다.\n", "- Autoglueon을 이용한 학습: Training Step 이용\n", "- Hyper Parameter Optimizer를 이용한 최적화: Tuning Step 이용\n", "- 검증데이터 추론: Processing Step 이용\n", "- 학습결과 Evaluation하기: Processing Step 이용\n", "- Condition 확인하여 모델 등록하기: Condition Step 이용" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", "import boto3\n", "import sagemaker\n", "import sagemaker.session\n", "\n", "from sagemaker.estimator import Estimator\n", "from sagemaker.inputs import TrainingInput\n", "from sagemaker.model_metrics import (\n", " MetricsSource,\n", " ModelMetrics,\n", ")\n", "from sagemaker.processing import (\n", " ProcessingInput,\n", " ProcessingOutput,\n", " ScriptProcessor,\n", ")\n", "from sagemaker.sklearn.processing import SKLearnProcessor\n", "from sagemaker.workflow.conditions import ConditionLessThanOrEqualTo\n", "from sagemaker.workflow.condition_step import (\n", " ConditionStep,\n", ")\n", "from sagemaker.workflow.functions import (\n", " JsonGet,\n", ")\n", "from sagemaker.workflow.parameters import (\n", " ParameterInteger,\n", " ParameterString,\n", ")\n", "from sagemaker.workflow.pipeline import Pipeline\n", "from sagemaker.workflow.properties import PropertyFile\n", "from sagemaker.workflow.steps import (\n", " ProcessingStep,\n", " TrainingStep,\n", ")\n", "from sagemaker.workflow.model_step import ModelStep\n", "from sagemaker.model import Model\n", "from sagemaker.workflow.pipeline_context import PipelineSession\n", "\n", "from sagemaker.workflow.steps import (\n", " ProcessingStep,\n", " TrainingStep,\n", " CacheConfig\n", ")\n", "\n", "# Additional dependencies\n", "from sagemaker.processing import FrameworkProcessor\n", "from sagemaker.workflow.conditions import ConditionGreaterThanOrEqualTo\n", "from sagemaker import image_uris\n", "\n", "from ag_model import (\n", " AutoGluonTraining,\n", " AutoGluonInferenceModel,\n", " AutoGluonTabularPredictor,\n", " AutoGluonFramework\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "sagemaker_session = sagemaker.session.Session()\n", "region = sagemaker_session._region_name\n", "role = sagemaker.get_execution_role()\n", "default_bucket = sagemaker_session.default_bucket()\n", "boto_session = boto3.Session(region_name=region)\n", "sagemaker_client = boto_session.client(\"sagemaker\")\n", "pipeline_session = PipelineSession(\n", " boto_session=boto_session,\n", " sagemaker_client=sagemaker_client,\n", " default_bucket=default_bucket,\n", " )" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "!aws s3 sync ../data/dataset s3://{default_bucket}/autogluon/dataset\n", "!aws s3 sync ../config s3://{default_bucket}/autogluon/config" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "model_package_group_name=\"FraudPackageGroup-Dev\"\n", "pipeline_name=\"FraudPipeline-Dev\"\n", "base_job_prefix=\"FraudPipeline-Dev\"\n", "processing_instance_type=\"ml.m5.xlarge\"\n", "training_instance_type=\"ml.m5.xlarge\"" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "BASE_DIR = os.getcwd()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# parameters for pipeline execution\n", "# processing_instance_count = ParameterInteger(name=\"ProcessingInstanceCount\", default_value=1)\n", "model_approval_status = ParameterString(\n", " name=\"ModelApprovalStatus\", default_value=\"PendingManualApproval\"\n", ")\n", "input_data = ParameterString(\n", " name=\"InputDataUrl\",\n", " default_value=f\"s3://{default_bucket}/autogluon/dataset\",\n", ")\n", "config_data = ParameterString(\n", " name=\"ConfigDataUrl\",\n", " default_value=f\"s3://{default_bucket}/autogluon/config\",\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# training step for generating model artifacts\n", "hyperparameters = {\n", " \"config_name\" : \"config-med.yaml\"\n", "}\n", "\n", "max_run = 1*60*60\n", "\n", "use_spot_instances = False\n", "if use_spot_instances:\n", " max_wait = 1*60*60\n", "else:\n", " max_wait = None" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "base_path = f\"s3://{sagemaker_session.default_bucket()}/{base_job_prefix}/FarudTrain\"\n", "\n", "ag_estimator = AutoGluonTraining(\n", " entry_point=\"autogluon_starter_script.py\",\n", " source_dir=f\"{BASE_DIR}/src\",\n", " role=role,\n", " # region=region,\n", " sagemaker_session=pipeline_session,\n", " output_path=base_path,\n", " code_location=base_path,\n", " hyperparameters=hyperparameters,\n", " instance_count=1,\n", " instance_type=training_instance_type,\n", " framework_version=\"0.4\",\n", " py_version=\"py38\",\n", " max_run=max_run,\n", " use_spot_instances=use_spot_instances, # spot instance 활용\n", " max_wait=max_wait,\n", " base_job_name=f\"fraud-train\", # base_job_name 적용\n", " # disable_profiler=True\n", ")\n", " \n", "data_channels = {\n", " \"inputdata\": TrainingInput(s3_data=input_data),\n", " \"config\": TrainingInput(s3_data=config_data)\n", "}\n", "\n", "step_train = TrainingStep(\n", " name=\"TrainFraudModel\",\n", " estimator=ag_estimator,\n", " inputs=data_channels\n", " # cache_config=CacheConfig(enable_caching=True, expire_after='T12H')\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pipeline = Pipeline(\n", " name=pipeline_name,\n", " parameters=[\n", " training_instance_type,\n", " model_approval_status,\n", " input_data,\n", " config_data\n", " ],\n", " steps=[step_train],\n", " sagemaker_session=pipeline_session,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pipeline.upsert(role_arn=role)\n", "execution = pipeline.start()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# processing step for evaluation\n", "image_uri = image_uris.retrieve(\n", " \"autogluon\",\n", " region=region,\n", " version=\"0.4\",\n", " py_version=\"py38\",\n", " image_scope=\"training\",\n", " instance_type=processing_instance_type,\n", ")\n", "\n", "script_eval = FrameworkProcessor(\n", " AutoGluonFramework,\n", " framework_version=\"0.4\",\n", " role=role,\n", " py_version=\"py38\",\n", " image_uri=image_uri,\n", " instance_type=processing_instance_type,\n", " instance_count=1,\n", " base_job_name=f\"{base_job_prefix}/FraudEval/fraud-eval\",\n", " sagemaker_session=pipeline_session,\n", ")\n", "\n", "\n", "step_args = script_eval.run(\n", " code=\"autogluon_evaluation.py\",\n", " source_dir=f\"{BASE_DIR}/src\",\n", " inputs=[\n", " ProcessingInput(\n", " source=input_data, \n", " input_name=\"test_data\", \n", " destination=\"/opt/ml/processing/test\"\n", " ),\n", " ProcessingInput(\n", " source=step_train.properties.ModelArtifacts.S3ModelArtifacts, \n", " input_name=\"model_weight\", \n", " destination=\"/opt/ml/processing/model\"\n", " )\n", " ],\n", " outputs=[\n", " ProcessingOutput(\n", " source=\"/opt/ml/processing/output\", \n", " output_name='evaluation'),\n", " ],\n", " wait=False\n", ")\n", "\n", "evaluation_report = PropertyFile(\n", " name=\"FraudEvaluationReport\",\n", " output_name=\"evaluation\",\n", " path=\"evaluation.json\",\n", ")\n", "\n", "step_eval = ProcessingStep(\n", " name=\"EvaluateFraudModel\",\n", " step_args=step_args,\n", " property_files=[evaluation_report],\n", ") " ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pipeline = Pipeline(\n", " name=pipeline_name,\n", " parameters=[\n", " training_instance_type,\n", " model_approval_status,\n", " input_data,\n", " config_data\n", " ],\n", " steps=[step_train, step_eval],\n", " sagemaker_session=pipeline_session,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pipeline.upsert(role_arn=role)\n", "execution = pipeline.start()" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# register model step that will be conditionally executed\n", "model_metrics = ModelMetrics(\n", " model_statistics=MetricsSource(\n", " s3_uri=\"{}/evaluation.json\".format(\n", " step_eval.arguments[\"ProcessingOutputConfig\"][\"Outputs\"][0][\"S3Output\"][\"S3Uri\"]\n", " ),\n", " content_type=\"application/json\"\n", " )\n", ")\n", "model_image_uri = image_uris.retrieve(\n", " \"autogluon\",\n", " region=region,\n", " version=\"0.4\",\n", " py_version=\"py38\",\n", " image_scope=\"inference\",\n", " instance_type=\"ml.m5.large\",\n", ")\n", "\n", "base_path = f\"s3://{sagemaker_session.default_bucket()}/{base_job_prefix}/FraudModel\"\n", "model = AutoGluonInferenceModel(\n", " source_dir=f\"{BASE_DIR}/src\",\n", " entry_point=\"autogluon_serve.py\",\n", " model_data=step_train.properties.ModelArtifacts.S3ModelArtifacts,\n", " instance_type=\"ml.m5.large\",\n", " role=role,\n", " sagemaker_session=pipeline_session,\n", " framework_version=\"0.4\",\n", " py_version=\"py38\",\n", " predictor_cls=AutoGluonTabularPredictor\n", ")\n", "step_args = model.register(\n", " content_types=[\"text/csv\"],\n", " response_types=[\"text/csv\"],\n", " inference_instances=[\"ml.t2.medium\", \"ml.m5.large\"],\n", " transform_instances=[\"ml.m5.large\"],\n", " model_package_group_name=model_package_group_name,\n", " approval_status=model_approval_status,\n", " model_metrics=model_metrics,\n", ")\n", "step_register = ModelStep(\n", " name=\"RegisterFraudModel\",\n", " step_args=step_args,\n", ")\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "# condition step for evaluating model quality and branching execution\n", "cond_gte = ConditionGreaterThanOrEqualTo(\n", " left=JsonGet(\n", " step_name=step_eval.name,\n", " property_file=evaluation_report,\n", " json_path=\"classification_metrics.accuracy.value\"\n", " ),\n", " right=0.9,\n", ")\n", "step_cond = ConditionStep(\n", " name=\"CheckAccuracyFraudEvaluation\",\n", " conditions=[cond_gte],\n", " if_steps=[step_register],\n", " else_steps=[],\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pipeline = Pipeline(\n", " name=pipeline_name,\n", " parameters=[\n", " training_instance_type,\n", " model_approval_status,\n", " input_data,\n", " config_data\n", " ],\n", " steps=[step_train, step_eval, step_cond],\n", " sagemaker_session=pipeline_session,\n", ")" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "pipeline.upsert(role_arn=role)\n", "execution = pipeline.start()" ] } ], "metadata": { "instance_type": "ml.t3.medium", "kernelspec": { "display_name": "Python 3 (Data Science)", "language": "python", "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:ap-northeast-2:806072073708:image/datascience-1.0" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.10" } }, "nbformat": 4, "nbformat_minor": 4 }