{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "100220bb-1ec7-4351-9883-966f1526340e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Collecting kfp==1.8.4\n",
      "  Downloading kfp-1.8.4.tar.gz (252 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m252.2/252.2 kB\u001b[0m \u001b[31m34.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting absl-py<=0.11,>=0.9\n",
      "  Downloading absl_py-0.11.0-py3-none-any.whl (127 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m127.8/127.8 kB\u001b[0m \u001b[31m29.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: PyYAML<6,>=5.3 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (5.4.1)\n",
      "Requirement already satisfied: google-cloud-storage<2,>=1.20.0 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (1.44.0)\n",
      "Requirement already satisfied: kubernetes<19,>=8.0.0 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (18.20.0)\n",
      "Requirement already satisfied: google-api-python-client<2,>=1.7.8 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (1.12.11)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.1 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (1.35.0)\n",
      "Requirement already satisfied: requests-toolbelt<1,>=0.8.0 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (0.9.1)\n",
      "Collecting cloudpickle<2,>=1.3.0\n",
      "  Downloading cloudpickle-1.6.0-py3-none-any.whl (23 kB)\n",
      "Requirement already satisfied: kfp-server-api<2.0.0,>=1.1.2 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (1.8.2)\n",
      "Requirement already satisfied: jsonschema<4,>=3.0.1 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (3.2.0)\n",
      "Requirement already satisfied: tabulate<1,>=0.8.6 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (0.8.10)\n",
      "Collecting click<8,>=7.1.1\n",
      "  Downloading click-7.1.2-py2.py3-none-any.whl (82 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m82.8/82.8 kB\u001b[0m \u001b[31m19.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: Deprecated<2,>=1.2.7 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (1.2.13)\n",
      "Requirement already satisfied: strip-hints<1,>=0.1.8 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (0.1.10)\n",
      "Requirement already satisfied: docstring-parser<1,>=0.7.3 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (0.14.1)\n",
      "Requirement already satisfied: kfp-pipeline-spec<0.2.0,>=0.1.10 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (0.1.16)\n",
      "Requirement already satisfied: fire<1,>=0.3.1 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (0.4.0)\n",
      "Requirement already satisfied: protobuf<4,>=3.13.0 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (3.20.1)\n",
      "Requirement already satisfied: uritemplate<4,>=3.0.1 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (3.0.1)\n",
      "Requirement already satisfied: pydantic<2,>=1.8.2 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (1.8.2)\n",
      "Requirement already satisfied: typing-extensions<4,>=3.10.0.2 in /opt/conda/lib/python3.8/site-packages (from kfp==1.8.4) (3.10.0.2)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.8/site-packages (from absl-py<=0.11,>=0.9->kfp==1.8.4) (1.16.0)\n",
      "Requirement already satisfied: wrapt<2,>=1.10 in /opt/conda/lib/python3.8/site-packages (from Deprecated<2,>=1.2.7->kfp==1.8.4) (1.14.1)\n",
      "Requirement already satisfied: termcolor in /opt/conda/lib/python3.8/site-packages (from fire<1,>=0.3.1->kfp==1.8.4) (1.1.0)\n",
      "Requirement already satisfied: google-auth-httplib2>=0.0.3 in /opt/conda/lib/python3.8/site-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.4) (0.1.0)\n",
      "Requirement already satisfied: httplib2<1dev,>=0.15.0 in /opt/conda/lib/python3.8/site-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.4) (0.20.4)\n",
      "Requirement already satisfied: google-api-core<3dev,>=1.21.0 in /opt/conda/lib/python3.8/site-packages (from google-api-python-client<2,>=1.7.8->kfp==1.8.4) (2.8.2)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /opt/conda/lib/python3.8/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.4) (4.2.4)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.8/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.4) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.8/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.4) (4.7.2)\n",
      "Requirement already satisfied: setuptools>=40.3.0 in /opt/conda/lib/python3.8/site-packages (from google-auth<2,>=1.6.1->kfp==1.8.4) (62.3.4)\n",
      "Requirement already satisfied: requests<3.0.0dev,>=2.18.0 in /opt/conda/lib/python3.8/site-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.4) (2.28.0)\n",
      "Requirement already satisfied: google-cloud-core<3.0dev,>=1.6.0 in /opt/conda/lib/python3.8/site-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.4) (2.3.1)\n",
      "Requirement already satisfied: google-resumable-media<3.0dev,>=1.3.0 in /opt/conda/lib/python3.8/site-packages (from google-cloud-storage<2,>=1.20.0->kfp==1.8.4) (2.3.3)\n",
      "Requirement already satisfied: pyrsistent>=0.14.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema<4,>=3.0.1->kfp==1.8.4) (0.18.1)\n",
      "Requirement already satisfied: attrs>=17.4.0 in /opt/conda/lib/python3.8/site-packages (from jsonschema<4,>=3.0.1->kfp==1.8.4) (21.4.0)\n",
      "Requirement already satisfied: urllib3>=1.15 in /opt/conda/lib/python3.8/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.4) (1.26.9)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.8/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.4) (2022.6.15)\n",
      "Requirement already satisfied: python-dateutil in /opt/conda/lib/python3.8/site-packages (from kfp-server-api<2.0.0,>=1.1.2->kfp==1.8.4) (2.8.2)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.8/site-packages (from kubernetes<19,>=8.0.0->kfp==1.8.4) (1.3.2)\n",
      "Requirement already satisfied: requests-oauthlib in /opt/conda/lib/python3.8/site-packages (from kubernetes<19,>=8.0.0->kfp==1.8.4) (1.3.1)\n",
      "Requirement already satisfied: wheel in /opt/conda/lib/python3.8/site-packages (from strip-hints<1,>=0.1.8->kfp==1.8.4) (0.37.1)\n",
      "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /opt/conda/lib/python3.8/site-packages (from google-api-core<3dev,>=1.21.0->google-api-python-client<2,>=1.7.8->kfp==1.8.4) (1.56.3)\n",
      "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /opt/conda/lib/python3.8/site-packages (from google-resumable-media<3.0dev,>=1.3.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.4) (1.3.0)\n",
      "Requirement already satisfied: pyparsing!=3.0.0,!=3.0.1,!=3.0.2,!=3.0.3,<4,>=2.4.2 in /opt/conda/lib/python3.8/site-packages (from httplib2<1dev,>=0.15.0->google-api-python-client<2,>=1.7.8->kfp==1.8.4) (3.0.9)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /opt/conda/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.1->kfp==1.8.4) (0.4.8)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /opt/conda/lib/python3.8/site-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.4) (2.0.12)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.8/site-packages (from requests<3.0.0dev,>=2.18.0->google-cloud-storage<2,>=1.20.0->kfp==1.8.4) (3.3)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /opt/conda/lib/python3.8/site-packages (from requests-oauthlib->kubernetes<19,>=8.0.0->kfp==1.8.4) (3.2.0)\n",
      "Building wheels for collected packages: kfp\n",
      "  Building wheel for kfp (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for kfp: filename=kfp-1.8.4-py3-none-any.whl size=349008 sha256=a993f281b982b48e7cc073b470be0cab54c8f0bb51c88c4ea55b76eac9c22e88\n",
      "  Stored in directory: /home/jovyan/.cache/pip/wheels/49/2b/fc/79b1a2787e4f73f9ee6b1c7fe88f7f42b82133c40baca38aa9\n",
      "Successfully built kfp\n",
      "Installing collected packages: cloudpickle, click, absl-py, kfp\n",
      "\u001b[33m  WARNING: The scripts dsl-compile, dsl-compile-v2 and kfp are installed in '/home/jovyan/.local/bin' which is not on PATH.\n",
      "  Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\u001b[0m\u001b[33m\n",
      "\u001b[0mSuccessfully installed absl-py-0.11.0 click-7.1.2 cloudpickle-1.6.0 kfp-1.8.4\n"
     ]
    }
   ],
   "source": [
    "!pip install kfp==1.8.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "859ce3ed-1ad0-4a4c-9c6f-1e7cb96a15b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import kfp\n",
    "from kfp import components\n",
    "\n",
    "from kfp import dsl\n",
    "from kfp import compiler\n",
    "\n",
    "from kubeflow.training.utils import utils\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "85707581-a264-4741-a657-9bdc9fcbe5c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "pytorch_job_op = components.load_component_from_url('https://raw.githubusercontent.com/kubeflow/pipelines/master/components/kubeflow/pytorch-launcher/component.yaml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "674c601f-04b3-49e4-9eeb-05c30a215086",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'kubeflow-user-example-com'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "kanwal_namespace = utils.get_default_target_namespace()\n",
    "kanwal_namespace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ca3b9800-86fa-4420-9967-3f88da022610",
   "metadata": {},
   "outputs": [],
   "source": [
    "@dsl.pipeline(name=\"PyTorch Training pipeline\", description=\"Sample training job test\")\n",
    "def pytorch_cnn_n_b_yaml(\n",
    "    namespace=kanwal_namespace\n",
    "):\n",
    "    \n",
    "    train_task = pytorch_job_op(\n",
    "        name='pytorch-cnn-dist-file-c3', \n",
    "        namespace='kubeflow-user-example-com', \n",
    "        master_spec='{ \\\n",
    "          \"replicas\": 1, \\\n",
    "          \"restartPolicy\": \"OnFailure\", \\\n",
    "          \"template\": { \\\n",
    "            \"metadata\": { \\\n",
    "              \"annotations\": { \\\n",
    "                \"sidecar.istio.io/inject\": \"false\" \\\n",
    "              } \\\n",
    "            }, \\\n",
    "            \"spec\": { \\\n",
    "              \"containers\": [ \\\n",
    "                { \\\n",
    "                  \"name\": \"pytorch1\", \\\n",
    "                  \"image\": \"763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.12.0-gpu-py38-cu116-ubuntu20.04-e3\", \\\n",
    "                  \"args\": [ \\\n",
    "                    \"python\", \\\n",
    "                    \"./efs-shared/pipeline/mnist.py\", \\\n",
    "                    \"--epochs\", \"5\", \\\n",
    "                    \"--seed\", \"7\", \\\n",
    "                    \"--log-interval\", \"60\" \\\n",
    "                  ], \\\n",
    "                  \"resources\": { \\\n",
    "                    \"limits\": { \\\n",
    "                      \"nvidia.com/gpu\": 1 \\\n",
    "                    } \\\n",
    "                  }, \\\n",
    "                  \"volumeMounts\": [ \\\n",
    "                    { \\\n",
    "                      \"mountPath\": \"/efs-shared\", \\\n",
    "                      \"name\": \"efs-shared\" \\\n",
    "                    } \\\n",
    "                  ] \\\n",
    "                } \\\n",
    "              ], \\\n",
    "              \"volumes\": [ \\\n",
    "                { \\\n",
    "                  \"name\": \"efs-shared\", \\\n",
    "                  \"persistentVolumeClaim\": { \\\n",
    "                    \"claimName\": \"efs-pvc\" \\\n",
    "                  } \\\n",
    "                } \\\n",
    "              ] \\\n",
    "            } \\\n",
    "          } \\\n",
    "        }', \n",
    "        worker_spec='{ \\\n",
    "          \"replicas\": 1, \\\n",
    "          \"restartPolicy\": \"OnFailure\", \\\n",
    "          \"template\": { \\\n",
    "            \"metadata\": { \\\n",
    "              \"annotations\": { \\\n",
    "                \"sidecar.istio.io/inject\": \"false\" \\\n",
    "              } \\\n",
    "            }, \\\n",
    "            \"spec\": { \\\n",
    "              \"containers\": [ \\\n",
    "                { \\\n",
    "                  \"name\": \"pytorch2\", \\\n",
    "                  \"image\": \"763104351884.dkr.ecr.us-west-2.amazonaws.com/pytorch-training:1.12.0-gpu-py38-cu116-ubuntu20.04-e3\", \\\n",
    "                  \"args\": [ \\\n",
    "                    \"python\", \\\n",
    "                    \"./efs-shared/pipeline/mnist.py\", \\\n",
    "                    \"--epochs\", \"5\", \\\n",
    "                    \"--seed\", \"7\", \\\n",
    "                    \"--log-interval\", \"60\" \\\n",
    "                  ], \\\n",
    "                  \"resources\": { \\\n",
    "                    \"limits\": { \\\n",
    "                      \"nvidia.com/gpu\": 1 \\\n",
    "                    } \\\n",
    "                  }, \\\n",
    "                  \"volumeMounts\": [ \\\n",
    "                    { \\\n",
    "                      \"mountPath\": \"/efs-shared\", \\\n",
    "                      \"name\": \"efs-shared\" \\\n",
    "                    } \\\n",
    "                  ] \\\n",
    "                } \\\n",
    "              ], \\\n",
    "              \"volumes\": [ \\\n",
    "                { \\\n",
    "                  \"name\": \"efs-shared\", \\\n",
    "                  \"persistentVolumeClaim\": { \\\n",
    "                    \"claimName\": \"efs-pvc\" \\\n",
    "                  } \\\n",
    "                } \\\n",
    "              ] \\\n",
    "            } \\\n",
    "          } \\\n",
    "        }',\n",
    "        delete_after_done=False\n",
    "    )\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "19906c12-b3c9-465e-8668-1fb4cc8c81b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "kfp.compiler.Compiler().compile(pytorch_cnn_n_b_yaml, \"pytorch_cnn_n_b_yaml.zip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c843219e-b851-4dd9-b879-c4fe543f8b69",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}