{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import base64\n",
    "import boto3\n",
    "from datetime import datetime\n",
    "from io import BytesIO\n",
    "import json\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "import sagemaker\n",
    "from sagemaker.tensorflow import TensorFlowModel\n",
    "import tarfile\n",
    "import tensorflow as tf\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## sagemaker configuration\n",
    "sm_client = boto3.client('sagemaker')\n",
    "sm_session = sagemaker.Session()\n",
    "sm_role = sagemaker.get_execution_role()\n",
    "bucket = sm_session.default_bucket()\n",
    "s3_client = boto3.client('s3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load label dictionary\n",
    "with open(\"../data/label_dictionary.json\", \"r\") as f:\n",
    "    label_dictionary = json.load(f)\n",
    "with open(\"../data/inverted_label_dictionary.json\", \"r\") as f:\n",
    "    inverted_label_dictionary = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get model output location\n",
    "In the cell below, we will pull the trained model that was launched in the previous notebook. After training, the model is stored in S3, and its URI can be found in the SageMaker console by clicking on `Training Jobs` on the left-hand frame, then clicking on the training job, then finally scrolling down to the `Output` section where you will find `S3 model artifact` (see images below for reference).\n",
    "\n",
    "The S3 key of the model artifact will be assigned to the variable `s3key_model_artifact` in this notebook.\n",
    "\n",
    "<img src=\"../images/training_job_panel.png\" align=\"left\" width=\"200\"/>\n",
    "<img src=\"../images/training_job_list.png\" align=\"center\" width=\"650\"/>\n",
    "<img src=\"../images/model_output_s3uri.png\" align=\"center\" width=\"650\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Local inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "## get the output path of the training job\n",
    "# pull the path from the training job using the sagemaker client\n",
    "s3uri_model_artifact = sm_client.describe_training_job(\n",
    "    TrainingJobName='cinic-demo-horovod-2021-02-12-20-05-02-420')['ModelArtifacts']['S3ModelArtifacts']\n",
    "s3key_model_artifact = '/'.join(s3uri_model_artifact.split('/')[3:])\n",
    "\n",
    "# or find output path in the console using the instructions in the figure above\n",
    "base_prefix = 'distributed_training_demo/model'\n",
    "# s3key_model_artifact = f'{base_prefix}/cinic-demo-horovod-2021-02-12-20-05-02-420/output/model.tar.gz'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## download model\n",
    "local_model_dir = '../data/models'\n",
    "os.makedirs('../data/models', exist_ok=True)\n",
    "s3_client.download_file(bucket, s3key_model_artifact, local_model_dir +'/'+ 'horovod_model.tar.gz')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "## untar and load models\n",
    "tar_filepath = os.path.join(local_model_dir, 'horovod_model.tar.gz')\n",
    "extracted_dir = os.path.join(local_model_dir, 'horovod_model')\n",
    "with tarfile.open(tar_filepath) as tarred_file:\n",
    "    tarred_file.extractall(extracted_dir)\n",
    "model_path = os.path.join(extracted_dir, 'cinic10_classifier', '1')\n",
    "model = tf.keras.models.load_model(model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load test dataset for evaluation\n",
    "\n",
    "test_path = f'../data/sharded_tfrecords/test/'\n",
    "files = [os.path.join(r,file) for r,d,f in os.walk(test_path) for file in f]\n",
    "test_set = tf.data.TFRecordDataset(files)\n",
    "\n",
    "def _dataset_parser(value):\n",
    "    \n",
    "    # create a dictionary describing the features    \n",
    "    sample_feature_description = {\n",
    "        'image': tf.io.FixedLenFeature([], tf.string),\n",
    "        'label': tf.io.FixedLenFeature([], tf.int64),\n",
    "    }\n",
    "\n",
    "    # parse to tf\n",
    "    example = tf.io.parse_single_example(value, sample_feature_description)\n",
    "    \n",
    "    # decode from bytes to tf types\n",
    "    # NOTE: example key must match the name of the Input layer in the keras model\n",
    "    example['image'] = tf.io.decode_raw(example['image'], tf.uint8)\n",
    "    example['image'] = tf.reshape(example['image'], (32,32,3))\n",
    "    \n",
    "    # preprocess for resnset\n",
    "    # see https://www.tensorflow.org/api_docs/python/tf/keras/applications/resnet_v2/preprocess_input\n",
    "    example['image'] = tf.cast(example['image'], tf.float32)\n",
    "    example['image'] = tf.keras.applications.resnet_v2.preprocess_input(example['image'])\n",
    "    \n",
    "    # parse for input to neural network and loss function\n",
    "    sample_data = {'image_input': example['image']}\n",
    "\n",
    "    label = tf.cast(example['label'], tf.int32)\n",
    "    label = tf.one_hot(indices=label, depth=10)\n",
    "    \n",
    "    return sample_data, label\n",
    "\n",
    "test_set = test_set.map(_dataset_parser, num_parallel_calls=tf.data.experimental.AUTOTUNE)\n",
    "test_set = test_set.batch(128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "211/211 [==============================] - 41s 196ms/step - loss: 0.7631 - categorical_accuracy: 0.7422\n"
     ]
    }
   ],
   "source": [
    "## inference on test set\n",
    "_ = model.evaluate(test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Real-time endpoint inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "update_endpoint is a no-op in sagemaker>=2.\n",
      "See: https://sagemaker.readthedocs.io/en/stable/v2.html for details.\n",
      "Using already existing model: cinic-10-classifier\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "---------------!CPU times: user 25.3 s, sys: 2.39 s, total: 27.7 s\n",
      "Wall time: 8min\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## create model for batch transform (can also be used for real-time inference)\n",
    "endpoint_name = 'cinic-10-classifier'\n",
    "model_name = endpoint_name\n",
    "instance_type = 'ml.g4dn.xlarge'\n",
    "\n",
    "## if specifying the exact container, use \"image_uri\"\n",
    "# https://github.com/aws/deep-learning-containers/blob/master/available_images.md\n",
    "image_uri = '763104351884.dkr.ecr.us-east-1.amazonaws.com/tensorflow-inference:2.3.1-gpu-py37-cu102-ubuntu18.04'\n",
    "\n",
    "## if specifying only the framework version, for which the container is subject to change, use \"framework_version\"\n",
    "# https://sagemaker.readthedocs.io/en/stable/frameworks/tensorflow/sagemaker.tensorflow.html#tensorflow-serving-model\n",
    "framework_version = '2.3'\n",
    "\n",
    "## create model and deploy\n",
    "sm_model = TensorFlowModel(\n",
    "    model_data=f's3://{bucket}/{s3key_model_artifact}',\n",
    "    image_uri=image_uri,\n",
    "#     framework_version=framework_version,\n",
    "    source_dir='../source_directory/inference',\n",
    "    entry_point='inference.py',\n",
    "    role=sm_role,\n",
    "    sagemaker_session=sm_session,\n",
    "    name=endpoint_name,\n",
    ")\n",
    "\n",
    "sm_predictor = sm_model.deploy(\n",
    "    instance_type=instance_type,\n",
    "    initial_instance_count=1,\n",
    "    endpoint_name=endpoint_name,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "## setup to invoke endpoint\n",
    "sm_runtime_client = boto3.client('sagemaker-runtime')\n",
    "image_np = np.random.randint(low=0, high=255, size=(32,32,3))\n",
    "image_list = image_np.tolist()\n",
    "image_json = json.dumps(image_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "## invoke endpoint for inference\n",
    "response = sm_runtime_client.invoke_endpoint(\n",
    "    EndpointName=endpoint_name,\n",
    "    Body=image_json,\n",
    "    ContentType=\"application/json\",\n",
    ")\n",
    "endpoint_vector = json.loads(response['Body'].read().decode('utf-8'))['predictions'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "## local inference\n",
    "image_np = np.expand_dims(image_np, axis=0)\n",
    "local_vector = model.predict(tf.keras.applications.resnet_v2.preprocess_input(image_np))\n",
    "local_vector = local_vector.tolist()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All elements are within relative tolerance of 1e-05\n"
     ]
    }
   ],
   "source": [
    "## check where (if) the endpoint prediction and local prediction disagree\n",
    "rtol = 1e-5\n",
    "not_close_idx = np.where(np.isclose(endpoint_vector, local_vector, rtol=rtol) == False)[0]\n",
    "for i in not_close_idx:\n",
    "    print(\"Component {} does not agree:\".format(i))\n",
    "    print(\"\\tendpoint_vector: {}\".format(endpoint_vector[i]))\n",
    "    print(\"\\tlocal_vector: {}\".format(local_vector[i]))\n",
    "if not_close_idx.size < 1:\n",
    "    print(\"All elements are within relative tolerance of {}\".format(rtol))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## delete endpoint and model\n",
    "sm_predictor.delete_endpoint()\n",
    "sm_predictor.delete_model()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batch job inference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "........................................................................................................!\n",
      "CPU times: user 25.8 s, sys: 2.38 s, total: 28.2 s\n",
      "Wall time: 9min 7s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "env = {'SAGEMAKER_TFS_ENABLE_BATCHING': 'true',\n",
    "      'SAGEMAKER_TFS_BATCH_TIMEOUT_MICROS': '50000',\n",
    "      'SAGEMAKER_TFS_MAX_BATCH_SIZE': '16'}\n",
    "\n",
    "sm_model_transformer = sm_model.transformer(\n",
    "    instance_count=2,\n",
    "    strategy='SingleRecord',\n",
    "    instance_type='ml.p3.2xlarge',\n",
    "    max_concurrent_transforms=64,\n",
    "    output_path=f's3://{bucket}/distributed_training_demo/batch_transform_output/',\n",
    "    env=env,\n",
    "    assemble_with='Line')\n",
    "\n",
    "sm_model_transformer.transform(\n",
    "    data=f's3://{bucket}/distributed_training_demo/data/test/',\n",
    "    data_type='S3Prefix',\n",
    "    split_type='TFRecord',\n",
    "    content_type='application/x-tfexample',\n",
    "    wait=True,\n",
    "    logs=False,)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAIAAAACACAIAAABMXPacAABJt0lEQVR4nJ297ZYjSa4kZvjwCDKz+l6dfWQ9hd5Br7XS0eru6uxMV2WSEQ6YfgAeZPXM3KMjdk5PdRbJiPAPwGAwwOV//9/+VwAABFRJBQmQJJkEABEVVREFNCmkrN9DBQDJOedxzmfMQwTuvu37tg0RJzEnSYjpNnzsY+wmIsLMiIjIzBkZk5mZKet6gKiYqQ5RARQkkmC9AypUhYAi2vcOCEAmSBLJjGQmkyQpJEEISRIACIiIQhQQUjIRwUwCMFNVM1MRCpOkCFXE3XwMHy4ikYxkRBL9TYCQWY8RM+ac5zGfj/M45pzznDNjEunD7vfb5+f9ftvHMFV1gtcE1K0BPQPXs729iL++6h4ggIiI1L/l+qDI+7cQJCAECJCokfntQiRlfTEJSn2KrM9CpP6G/+J+/smrLlcXJyhC9DcQEPbXE2Bd6vrEb2NB4HVd8O0tXN+3fvj6Qrw96+tvXi9nZn1LghD2NPA/ebB/8lcCqCqpIlBVkd+mrabn7WYTTGbWgv0nF0hCEpBEbwqQzKwdICIqoIqsMUFvAfnHr/rLfV+bG4AIe+Ogp7h2gPyTWyKBWt6SIiLZ85T99GtuyGTvOv5loF6zg/cJpkec1yCm8HqIeiQpI1MfWKvwtQoAAQQUFYOKuCpMreagPqwKcm0OZg0XM5jBzLxGQ0QEZDLr5pIEKD1DycwEo2YaqiKStRlqwQiUIiqQa8u9r8g1AckEwewFAwr7YiQzExDVtr4CqXkhqYKUjAiCqlJWjcmefOmds6ze+t/rHpJkokxU9nMzSfWYJ34bdKxVJqIi0J6+a50QvWkELEMKmoiawVQUunxGj0TbJSgoJDIBkJERmZEJUgGBqAiZZUmDhKhSAVUAzMgIMkVgajAXWG+sWmu19WrJCJD9d33DazayRyZrk6sqyj8kIzIiIZCEUV6m77U3SBFlqghFLhvUZhLMRJIZmcn6H5lE1mUDmZmiKN90+TzPiGuP1PiLQkUgqqLX2l8X6xl4TRkFShURdRGIQiBQvbaP6ppaAEwECGbWBCQpXJut93pmziABpSigBjIzMiczRASWCklRFRICJkARoYBUUe35wOvWa1OVdSg3CYgRpKoIIZEZETNSRFQlU4kEpFYuMylgMidExGqfCV/WS+qhkKzRZeZCAK+pz2Bq4jU3VDL9zXLWn4RrKpAJXTZuAY2yD+sjWv9TVVVRq+GvsVgm+jKjTCZq/a8lkKBABGLXu64dClLWvGfvgBCRFKm3QORaHCKA1m6tO/qrHS87c1mA8hik1q1Gr4cUkcx/uHOQpEDKASyDSVnWoXx7vuZ4TQASr59aCVJTkmtb+Rhe18lGAdKTgLJSKYVa+p7L8JRtgQpr16uIqqnab/e0EEXZeSFnOVf0qqr3yZrUNovlHkhI78h1zwsvvcEQKVMilEKTC44RAmjd9DuIWgs0SJAmgCBJiWREREKtJ1EFqste9gM1lsie4QYsa6u9/4lvF+3/FKEKeqM23AIJ3/bRPorCRLsXBnKy0LewVpWsXVAwTtrnQSAiek3AMlW54NeFCuTaODXc1xi2y00mC8EHk6K1lbIeTCDAumDP3Ouxe2ooWVuqLZs0gnoFNr1A264wwchEUiILK7TXl341trp2hEAIyeUiaiGSQhFK++S/wNGyBhUq1AOUN+wJGON9AiQTmZnBlIpCKCrKsngviC8v7A8VKcdbXrAGVEQKz9eqZi+lXjRrbK71lYUgmMFGqASF1PVQazO8jcqavbacKIvFNV/Q13V4YUQuC5BIMoKpJJISFEALiQH1tK/749sSvyZ9RU61JK79eYHRwtlZflSF5Rx1IYcyOW5mjfSyHoAXgvxXL/n955/9+h9NcP1TFpuCFORrNigJ4bLRy8dwWU9dF9BCB//qxrKeSdDOpj/415tZ6IkkCoklkRRAU2zBMxXRHsd/cTmux37dADIz/vJTEQzehqueVCQBuIg2mMIy2Qv0sOZB1hKviPtt7fe/1z7t2yr3SWW5rtcdvlZPrUcUKJdamEIyIrhQ+4UBgVeksPxNIV0RvseoiZQk1v4QiEKldjFBLFNZpjSZEY0JMhEJIoCcU+Yp0xrsL8zOhfWX+QFXyCZl58jMjMg54zhjxsxZU8DITCLbATDJSE4kCXHWL3nZgiz/2Oatt42qLGi/HrmDT2luQJbPIS4D11EaICJ8YdELWzcsF6L+jQLq18Iq5yDCtj21DKwjjbL1Uu6mZjUCwl4RqqhxFwUUkhQFqOuuQaGCmZAFdEkwzhOnww1JAzpq6AXEZPlRucDia8OTTMaM45zPOeecmZO1ophkwVZizdNZns/f48W145INpQUdD6io6j8hGPpZ+urUtXTffrDsdEVIvYkuSH6ht4ZYKEv8YgnW1drLqHaU97bvro3Ngq4C0YrNkghKQtdeLOjQRERKQ4wau8wENUzmqadJMmvhsG1yzfZistaTL+NaIzsj5oxzxozMMm7rr19kSc1B+VV/B1E9orLGtm2orp9rw+F1V5czBQpfZ+1ccu1dkK+xUm1PcHma5R373kTfLtJf3wvAzFRF/roSXjOwZgEpmQ3nCvEAKCuUuJAxeV21V3djAc2MTCvUf01A3a3p5d5/WyL5ekUWlHjzpjXlaxm/1gsAX4AcDQJMRYimdRcSx9oNi4u8iM/LA1w7LV+EBX+L+8t9iogKqbBr4q5LreF+QZ3efLZeWmAChSJfZAcAiK2NjBk8Y0aWma1vMRBgMIKcrLgrLzqqCQsADbuXrQ2CxbmoiCCpAirKDwkqEH+53FpKgnqDqJk2XGq0LhWzrt0OZwau51ARKpUIARFZWH5x7uxo4Bp2EeFyQb3cswz6a/E1maWkiPZnlRAVSVFBM1QFxATNp2lfQEXUVM3M3c1MVHuVFbSVxfotuC5JCueMxzGfZzKhrsOHey0syYicB3Nmopz2FfhfD1XjU/6u5klEaNpr7jWaSiyCKTu9QQJSnJipqIgJtbBErV+zNqN1404G2slBRaFawDWTekXZZdkFUnMg8kb2NPouHrNvpm3KtcuZiTWy5VyEotIbPzI6tGpktYZirQq1tQdEizJt+6YCqV9eplAyyWPG4/v4ep6Z8G3bb7qLmggTOTPmRE4SAusISVDhzgWvarWUWSqKQgSZUhwq+jMVklbkFBlRzy0QFVM1ExP15X0BoQBaUGLZbydTLnPPd4vzejGZFcYu//iPrwQuXPOXCXiPK9qX10MkUyBFGCwb0OCuhmXdjGIFYS+HwysIE2j9qUKrc+bzOL8ez1/fz0hss4COuFF4UWQXcSqqStJMSehy9S/MdqHz39mty4vk76833yAFrnENgOqagEaUtQMaRBUMvOw3Fj4lkgQRhe4LhRNtxeqC+ebIWZRsURF19Wv4Loux0LsAWAi7OEdBJfdEdX2qTHShJanQsrGFXLaYkokZecx8Ps/vx/H19f3z6zuS2zmDZORwcZtGqJha3YeVR6lBIUJE3ExUIYWQVnhb4eK617JKtT0i5pwzrh1QcXSH2pXBUVVVcxW1cgMi0N7gvoaixjhqTPJ6yL5a2XExM1Fvh7YQ8IU4r3WRmZEBsAAMFmnUc3A92OX+EkwUOSNJCKgCbxQMJSuaASDCaJ5pgUpR0QCTnJPPI74f5/f349fX169fXzNybOeMyBn75rcdNxez4VZeQ+s+MtIKGoiYuaqJ6JUbkyLRllkECrgEgZ6Axaa+fHnxWREEVNR9DFExV7Vl5fqbnReY7cGovdzEcYOzcnoUCDQ1VbE25ItHWqDuhfBYvOnlJuVl3iis7NgbdVKmHZJFnUi7/GAqI5KC1Fp3jFxx3bKlRCTPGc9jPh7H1/fj6+vr19evGTHOmUmhMDdXo5VH10rV9S1rZiqZtaU6tbPcZm3DFxYlE8lErOxCR3EvGCUkMnKeJ8kyaG6jUJ3pwqICEfhv1E2H/n+hT16QmeWTkEhdqe3CoODbR174dw3+iiQuG0Mmcv28HEgmGAChUgRK5d6ESWG2ZUBxexcpVB4hMueM45jP43g8HjUHM2JsFNFhw00iBDBVczPVpu9rFa0YvyegEjuKtgQiLPB4DVRlLRZRsbaHyGKq23bUG8wSgOItlGyEIa4NIcBOAVVkJwsBXab65ZeLO75oYZQ9uCJzEVVFpZo6V9MXLLpYls150T0LbaxkQLIQNgmmZqapShQNxGKbVaDCbA5xJubMc87zeD6fz8fj8f399fXre2ZuU9zGvp17eNJFpDCVKiWJjMvMrzhECrADAFRaRcFGR2tNrs+IoKysykLgyWBldTrP3G+XVviYvsCzuFRsigIehBT/wqKdsuGrNoavJVG8F5dbAgntVYSVmnkDhmtDKCkZb/tomf7lB4IZjMkMCpABJtLUYpGBIqpiJmKkKaRorYq85ozzPM/jOJ6Px+Px/f34/n7MTFL3221GVL5f+ju091ITL2uTS8H4K+/YwwSk6goQ1iboNS8qNABGkpEZQlLftudaxY2o14qspe8q2l6gbQxy7YJrBxRu6hisQCl7r5VwhJ17ulbPWkJvqwtULsCZgQxGSyOWw6jZyGBMAVHZ3UzNeHlwVXFXGzBCFJFQpHIGZ+Scc86zp+E4ns8zSfe4aGpdX3Lh8JSXDmWB8F7VNbTRT61ruBeiF5EK7yshggpQihWI/riqaDaNJWX8a3ILiCoAx//PV83XP48JrjVfj/XuHbhkAjMRieKB3zJlfNsUKL7h5azrWXMzozgJhQYsE5qJyJxFhkVE1M5g2XR39zHGGOamuiKe3x3dddvvdnoBC2ApQET+sjhxRdAA5OIxgCKvQFGzYcPMdC3/3gE9Aa9L9IiWtV52OvEiANdNl79/UWLvoduL4Gn2Itsn1fKDiJAJmcSZzFRCISZqasGox8rlF9s5RoWaGSJiM81DtzSC6mqT0CQjEYHZzL6IqrlDdL/d7p8fnz9+fHx+7PtmaiQjQpQXnq1xJyF6kb8Vgaz4r29fGrFkklFQ/0oSlNONzMpuqto2hC5mNrbbNjZ3NzUVa2BSE/BGd8tv48t3X/m2Qhf6x6KFrgm4dBoJapRF7fQ+igSGCoSCQJ6ZR0iEKNWgqiYwqq4kfUftBSuYLOsCgc0wDy9NhW1mE+JJBBlEUAiDmvu27Wnunz8+//jjj3/793/748f9tolJZkYg5aWJaXKFXC6yvWT51abcdYVQCSCiAlR5sV7FGlVEQEDMfLiqmru779vY3YaZdeiy2CZfwLOJ8TIT8hIrrMl4rW5eJEFtv7cd0G8glrB3MapCrjgSkTiDz8nHmRl0YNPX1s96zBS1/lUyScyZ5zlJqKYPJuBiOE/ZQhYBSShFRd19G/t+A3xsnz8+f/zx448//vi875ulxZM5I1IkoRAFO/XXubY3OggFddnZB9MKoEkSKhcObBlwk9GMTFTQOkrN666+ubpao5OVQgEgvugLyL/Ota61z1dW4D9/dXwMLNI1CUgGJYFj5tdz/nocj8fMjE3BUWkALcPJdCj7dlURHSdEZJKahCg8EGElG1i+X83M3MYY+36LqWZjbB8fH/f7fd/3sW2GU6m8yPbkWkYir5RPb+SVxvj9LZWZeWGhV16pQf96s2rpqYe7NzD8PY9RC96ziVJUAqhXct8PJWv0WkVS4gss7UnPyrKQDYSwXBTawJYPiZkz84x8nPPn9/Hn1+PxPJB5G8ab625DxXSI00WBNDUxhzQ5l0BSstJuBFKEWmvY1CA6PLcxtm2/TUYkRM59uo/7/T7GVuyCQEUFqpLBpRRrKnQ5yGJKKrm0Rqr4ECwLvAL4VwZmBWToSM5UbcW82kuezb2AlTqsl7ckWBbrv7JhoioJUVm6iuY4QEWSWtNWxA07LyC95Nc9omMIgIlj5uOIr+P4fhx//vr+89evx+MpwMdtAz9MdwwTdVcRHyptFrKuKQpRiqZAoIAKTMvgiKuZqG0D2x63GYCIig8/I8z8fv8YPgomoiy6iYgygrGo5vqOxifIotKAl2lVNBt95dkyVx4ma2sCkI47zdRMrVI2HTiVdcc77QBAvCea7/uqKZaeA1mOtIaWgMgKtnmhiDXuWAvkwtWSQGQe5/z1ffz59f3z+/Hnz58/f/58PJ8mEvHhrtvmZUCGuSusXS8ZiSRUoQY1UFIU0IAqNLHwnNkY2LctIit9MLZtRoroftvdfNmUihRVIEwp4lZEaj1VrLeYl0uNIhBFQiSLwPpHCrozSiKmImJWaFetifeLcWyzIWtqCYi/APt/9lo494oNf2eQ3sQ7f4XXhU9m5OM4f309/v7z189f33/++vnr16/jeLqpqtxu++2Y7mMfCnd3NQOSMyI5oYQ6zGEBJqRGAy/mglTSVLbhmVu7gnOcM0R0bGWFm0nQlT3If5XZqId4cWv9PMIUpv3DA/5loBbQt8uZ9NflKjxp4NXj5apWFuOKst/YTBbZVnUXEPT3li8qDIbrn5rkCgWb26qcwAw+z/n9/fz569ff//7nn7++fn19PZ7fMWe4+Rjfz+P2PMYY99smNnzfhgkzcZ5BkaS6q2/qJGq7S0TIOec853nEeRRMHG6Q3TzVXO3QMwBx92aYlZ3k7CC23Hatr5YArWQR30au/rOkTln6pbYDlXGUZcAXB1A/K2NY/nIRbYBA2fBcIHRzl8vCXMqAmoHsCVhfdylwq7Rq5bDq07lEqwXd0IYoyGPG8zh/PR5//vz5t7///efX1+PxOOdZz3ac8/k8vp/Hvu8zAXPfb8MtYyZEg2IpPmykJYgzIhjByJhzHsd5PP0YomJjbB3yUm1CBXJmwny4m5maUstkEaIwE6ZeDEoTZ50paS0qucTMSEUmszQtTe2INafdpOQK49pL1ehXDUhDmIVZXrvP3RzE4v2Ja4Zf9UONiXXl63oCkim5clNlBxZ5vMKEJGfwnPHoHfD158+fP7++juMoolwUc85jns/zPM4ZSYqpb+oGMYmETVFTG+q0BCnBo4hgRsQ853nO8zBzc3NVsVJ7a3HbQZqamqmqVMEHoChGVcz0Pc+YGZ2w1nfDzCt/qm0Z6in10g5oCbx6Tyy0uvw3sTjjWvWqggKGUiZIr2CLzSe9LH2ZlA551/btOOLd9CWALFStnbxAqY5n4Jh5nPNxnN+P5/f39/f39zmnQNxNVOcKHyNzJoJSUqrEClVFKj5Qd01qpU4qhV8EZERmIFMFZqLQHJzhwYykiHujm4tNWcBP1bC44osF6KHX68HeXr/nPVBZnWuE5Rqcf3wt5YcI8t09uJr12CfWZmLPChbZ36zMZdJwGcYVc7EWCQqTlu4sMVNm4JxxnHGc83mez+N8nuecoSIUaKWPV9I3Mo8Zj2OSYFamdSE8VTOFG9OFTGmtRE0DlhRZSFWaynALjkhiifou9UxWYSogAlO9xq+2+3tUL+X+KtbjK8Bhpzqk9RJrpb9PgLzYuwtXJVmssl7+8pWa4xK2A5BMwVIH97ohAUmwOaxlolZCmFf5i3RtYyQi5JxSE3Cecc44Z84Zc0Zp3GKFsrV1I/g85tfjmJHCmeeMGTU/NVgw46AIMq14lh7dTupEyZxM4KYbPPN3ZWdmMpAhrIxO6ellxZlr4CBlxEVSuZT5rw3Q/ycv/Yzwn6x9WbERUGWGEUsyTGQphuUqUWqEfAGAztte0HfZ+mtTdvqfV4i3SNpVoZWJSK0dcEacEYsrzqpwblV0hY5mVZJ2zvh+nhGpEjJn5ylRK07hQFWElUDDXKqKj0vt28kDddXahcHOHBa3WDIwBS/+AhApKgFrVAvGLFBRY1LpUC548TbM/Zn23Gv/rC/qndHzlqS+inMJ+AoTFupd2+dVLf2f4t7Xi70zXjRJMyWcmTNy1tqPlq0WsaVL9ebu5i6ixRWfApeQzCtboqq9RM2K5ZPepLiqDnPJYzpW7gfPkutVodY711KehavahZG4cqhmQBtGrrTl65WX7K9HU9d4a4UaL26yx7OuCBMxK6KxBCMec8q1pxafwYsAWnPdRPjbNnu/pyt06KupLmhMgpmcEWdEbYW56jJV1d3GGNu27fu+bZsPl2WUs0OTWl3KpXMX1Ss9QVRRZcgMH6txgFxK8AAhDAEVWWxpCfsMYqZuXvTknMG4+iCIu5k72ooyyrUtmqXsbdd5sn1JJ9BWjGSyVPW1/FXdh5oBqubqQ3VULtnnecqFCt4G85L/gehqoytf9wqll2Va2GKF+qYqAUh07VFkRpb9icwsQOtu2xj7tt1u++122/d9jM3NXqINWYkLphSevhzkChYzqIRIRiwsLtrYPQJMYQookkAKYK1AppuZuaqjrVcziaY63Hw4BHMCqGLmlyEhwKz0Z5Cpam5q9kp1dtgqkOJ3RExWmrzUQTZEHaIi8Jhnz3ynoX8rdmWJ695c+19W/kK5y471vOurjHft16UB6bU/3LZtu0a/XmMUNwFdo79sqnZchK7A4EsEnEmIFl56IU2sVQRWaUpWjZyyKu2hrY1QJrVl7ykiZuruY5QujJkaLQFdHjXL30dUIxIXrCTCpblVQSWopEPm3njrx0W8SqirV0RdqnQw/+TVJa1YUcFv9g3A0hAnRUQ7XpEVKb/soJm52TbGcNv3/bbv99vrtW3bGCaQWrNF2ih7W65/9RRU5F285e/kGFPbFFUCqxVlBK66vZ7by1223YdZ3WRlb1GIXdXa/bw5t8sH8BUZdABWQy2/rdNy+TV+r/iMxBJm9de+KO+mOC91aF6ajkqMo/NmvQO6vHQmVsxMqYLL9gyVGd+37eN+G27bGJ8f9x+fnx8fHx/3e9mffdvNVZhgzQGkNcG9qghoMiKAFEmsgrK6hWyV5tSqGTd3tRIPZs6lya1Qphj/8nRtpt0dZiJa+RM02BdToztEC5HnVcO11hVWEFMl+pf6s2tj2ArkEhtopmiqUjVFlALX36KsV6BXc5mljM2IDJBNP61ivcWz9uNHBC43AIlcJV8CM9vG+LjtOT8iYhvj4+Pjx+ePHz9+fHx8VsZq2zdTEQYyejlkF1uXbl6gmSUh72slsyrnq5h+VqceM1dTH0MVYMYZEzkjCw1V7XNX8aCiThX1MZaUREW09FSAmBocIjEjY2ar94gLqqIFRkjS2t6oCsBmavMq0gEW5B5VPQCB17dca32Bi56UDnEyy9616G3FKW1pmYwopMl2nkrRqAQWq7xEN/f7viMimdsYH58fP358/vj8/Pi41wTs26YCULDEWzVWpfAQ1QrFMhUdQCEzuOApmTHnPE81Uxsmqj7KjTBnrqV6BY8iS9IAqDcartFgI3aS3X1HoOTMNUrXMitajpBXZd0bXdPC7ipVzcweQKPT1Coodl21tD3q+RZl1A1exrXFtlkl780I5VJjxKwJSJE0Ryapb/tJzWyMkfsAse37x/3+8XH/+Pj4+GhHsI2hkgikJEITTLZ6uayKiJU2yBZXFpkVTGPxyXNOnVM9tLN2CCKSESxR68t3Le7rkhCXZCG7xmHRl9XHpRqYvL20glbiYs2uv2mSjvHylS+cHkxkqLB9matq4RhJvOrqrtu7RJu972qv4KIgMllD3xMgSNW+fbADmRXit8Adsm1jL/Bzv93u9/ICm7swUhjUZpRUK420lDqGUpSvKsksPdBFN5YjOKfahJ+EiiDmnGfMGcjUKgUoUeAr5/XyfvWnbHDFtdMvk9Bxr/bdXQG1XGmSF3rrLH8NQxVj58usoCFhiXObyawFfymGWfFGLhdMoNxW8mV9au3HzHlGhECuULuzZq8ou2R5qirDvQDo/X7r9b/vwx1xBkJSs5oUsFwbIYqKMtfjV8xT6wNdI9+l3jgn9EixCAIScczjiPMUxjBWTl5VkVgK1xX6dLcUdg0aX4N22R0V1uVLwN3b215iFsjVgEdFUxtDC0RkNcVbiRoCcPQv15Dmsr69DOpm2npeGdOyuYiSclZxSDCiLC4abVzXW4obMzN1s20bt9tWI3+/74WB3AyTZxpUYhHtXcuhVuq5tntaqkVYZkRqVmMYkIwZyTOhk6IzSESc83zmeagkhpgqmgLiq8J7TQFWFVOZAiVSr+qtYk9lFfIgVyFVMbWvMAwL14isHjr98YrnXiafcEZAsGxPw+2rfvMNXb+TD7zIxWWeFjW64FTDVRUzMWNJM3MME7rZ/bbf2wfU8t+2bZgoEXk2TKubASuRaFAXNVTvml68FDVVV+2AO1kV4DNSGAk9SJznMc9nztMUvJvZtnlpt3Gtek1r2vuKHJNkljG/TLupwkBI9AjyNQErW/UP/gBVa9MF9q9RbKGEn+eJq6fUq4SN1+jzL419VujBtV+47uNKmXVgqWJuDmwwqoGhiDhtuH1+fvz4/Pjj8+Pz4+OjOAh3BSKvZaSirUiDiJiLDVFnV+Il5kQm1GAmZkoE0QW7jOSc0CBmxPM4zuMR89xcc97ceNtuTiFzZsQMLR1puXJol0h0jHWRfmVEREscI0J0qVQF1vaSnS+fLNJMKKXNUg8kOsYAAPjxPFDSnnrXavrSfmjtjGYlL/eLZgIWM14PAVEzt8ocQ83NN9U0iIcIXJBzjGF//Pj848fnj8/Pz4/7/bZvY7ipFGLtqq9izcudmfpQ38ScxIxInpRMlLC3FCuUzCQys7IOzxnPM45zfj0ez+cjY942E/6432x+jOFSkOk8T1VV81ELSVZ801PQ1dEqCoOZ0aAR7Sokq+DZTNzk3QRdJlheoftrWi/UsHaASIUIJVm6PGzvgOjq08VE5SJciN+3oRST0h5JxcTFNrNMUw8FhpDhw+3z4/Pz4+Pz4/5xu+37NoaZKlc57kXsioiUcNo3HZuYZyJ4EhGJGS0a4mrpldm5+ufz/H6eX8/j8Ty+Ht+PxyMi7rdx3+15fJSwokPkCC319eJysUiBZZKbFOhCChNRITmnRpViqLhKcVi1YmpgpJervL5zzQHeAl6/pmLBXgCLYa/USXWduF5m+MdXqbhI6aS9qIIqrpq0UTlx0pEM2za7f9zu99v9ViT0MPdia8p0QVWoSqJFGCbuat4N/IgZcZzncZ45z8y6yW57WOm2OYuVqFeVMXbauXZ1rhzYy/9mVjO/vBwb3vifNQ/XYrs4ol5yqqr6hhDXh/8pufb2umrE5LV9CovOGWdpQEguLr7SUu3JF6dVqLjf02mgQozVEsK7vGSEUDi2zW/3+16ut1RTZmjRzOUArFh0NRc1sSHuhFYPnuM4H4/n8/nMOdmFTRGV8Tmjmg4Vq7MRJRaPzH33MbbOo7d9aeYcABkRk8xIskthy1Cs2HmRuWQCaQq6EtIyRDMRqczB4jiwiujqz2VWm8Nd5pzuZh0nqRJa9HNWA+Rz5jzZ6EBUTUXQuYGKEZUAshzNcsWVjSrGT2iAA6Iq7qYQYN/GXtznNq4OEPWkF5eoqDXm5gNiUIdokBF5HMf39+PXr6/n45kxu1VCBKs/85yRCREfLm4+NvOxnZOZ+673+93dl060GGmWqWPmnIeqXomQC67XMEZlNAuEkaoYboAszUvVgC6GPIv5Y/9TJdNFYgqzK1BAwM3LpAihiW4kVhXglUFhcxxFkggWU4oL3MrKnbDbIa0dUCKc1ArjBDRVURvuw829+5+UyqjvAVJU3uKtpPd1xszjjK/H89evr58/f/359z+P41m5HVEBEzOZE1G321DACfdtiwC5DbndtuFexFeNiFvxRUJWre/qiksBtKFQw5eoFgCrqYjAFKhlKSIrebpMGiuBvlx7mYvVdBeX4MXN/WpMyPyNxu6ouZbyvzBmfLdy+lYJW5WqSGS1sFh9SdQylVlZqla1N+DKS4LRDxLJYjhLq/L1eP76/v7zb3//2//8+6+ffxZ8GG7mrgJJYhXaX9x71fSJmRDbJmMU9XLdr6JqfDvpUAtc1pJF5SayV1enntdytldAW5m57P7SF5H3/+XlbiNbigyS1S9bRNRMzQFI943Aq3RkjX2+AgFBdWwoSOyu5qUeyTPirIIkBgnVmLfdZX7cmLeKqSIRkRFdJCwXwz0Zkd/P4+8/v/72959/+/PPn7++f/389fX19Xw+SY7hFch5EWm9aKodXi62XyuQ8KWQ6ygbkl35xddDLazOBCXl9wmoV+1sInURAutfWPap1vMKiZe3X28B0CYEAjcfnZ7IlMiGTiLmDiLDqppUFqusHc9xZS5X35J2n1ZBk7onBfPM83l+H885H8c8Mil63j921+PzHuedY0BY6CUiMmb5vILzzzMfz+Nvf/78j//+//y3//u//8f/+B9//vx6PJ5zToDufr/f//gxQe77BhstVoeAzNmuWFTN/BUm9jtantP9F2pC2sFVHjObZ167qQeJ7ZpFug/zRRj0uIp1S58uvK5pXaRmD/+LMXDVIWRKJmd32K2xNgNF1KgTqciQVSlYgVzZjVWagEsmKWYNXYotnUccX+fzeD7PxwyKMuL5sZ/Hv8WcOQOacc55RkSAkRllkuacxzF//fr629/+/h//8d//63/9P/7P/+u//fnz15yhpmOM221Pcgzft61IJl2y5GREMDNEUs2rvP6iK7E814L4WFHHGrEVY5ayCB3LXq0NgLfWq0sfEcttdUHiWwqSVycltrajBNoQoWNVXPd+uejjC5hKecWXPgy1RF95AgBQK83X+lm0NViCtYmc1RBl8QVVXjKR0nA9Am8Z3isSjJjHcXx/f//69fX19ZXJbd/cXX4b0Au/lnS+ELlcNgkVM//+ku4Z82qhRDIzgLfqpezIkJ32Wp/9Z+3nCsBLpmj1/f0thfAW6pUTB0Scr7nssyRWiiUYVRS/ZmNNQHZ2NLsxJpq46ZYVfVWphVMclpu5c0hSbQz3ArQZGZOUmGecM6IyYeCaVVMdw7Yxtq20Q9uMENH7x/3z4+N+v31+fHze77dt4dl2UwAV5i1OsK7OuEahzPEKe6RSyC8wmgkhEQSF1l37AOKV4q6suaypEFW7frPyPAIKstHjInF6y72CLvFL2nHRsFX8VApCJAU1tiv/SQCMZCWBI4IAzFhGtBV8a0+JqJmPMSBUr+B2u922bTMVMiNmJuY855wRea2jkoqMIbfcPz8+/vjjx7/9+7//ejz32919/Pjx+cePz9tek1JlAb50oqvPk2pmkywV4uSroSCS1NWKT00Ks4pUHrvYoMgWY63Qp/u4YpkXUthpZIfAVqNXNFLiOtHgteNe2RvV7jTvFfnl1fUvlmWJzAiQ9r74l3iuZ2jOjKjnWDkYIeTalVKE6DaorhsHpCZg3zdzIzNmROY8z3lGvCUXAYjqcIVsn+f9j3/747/8l//ljHg+z23bisvbxrhAwTJHekWIql2EumKYCjC7oD6MloCV1RS1Ku8SQKbiSvFm6TOYgJK4OPkXMdnBtLRIMkot18S29JEwcrFyi1lteykQn7PUq2/S2fLpKxRcprUS/cRvPTIzMylihQtWQJjVQ7Q5PvdBMbgISy2y77fbzcwARMwZeZ7nec6MXGoYQYEJVRPs+/b58fHv//5vM3icc9u2olGrPKXOLAJXEzi5GJyLym+rmFUjPvN5rsjI1Ox632KHlq0mCoumIEVaZ7UMbE/Bm7ssC9FtroqHWBFqBRS/+YPr5edxXBNwzjPOufLvnTdSEXvp3CvWYL62TJbCqxJealZkYS52V92GyoCJu47NxmZjbMPNncTMOLq3yczMYgfUTBevZGK15GdwjH1GmI193/dtAJzHeTwb8QBtJ2uQLqa84zJRABH5PIku5FczuEvD1kgC0eLJDom7nQ1n6X1ErUaTIDMIIpnSKveLHG7z2+okFREGwMwqcX4R3iDox/NRi7qaMcQ5Ca78oao2t4O2f7zyZYsn5WrHWsSWlg2JzF7I7ipiOny/+e0+tl2tXTCjztuqCTgzU9QcQ1QIW5kG9eGfn5/q4/PzR2RCCush5nwQ53n05QqnWbc0yuwihyIzVITAGYjkeeYMiNAHtw1Y+a/WNs1VtFBkWiSFooS6iokYBI1Osk4ruNpQrUDrBcjN3aTiQmq3uuqWhQXS4OdxAFg7YOYMAWS0gsGsZxzVp23Zn1jLn1eu5tpugu4AXEGPqrkP37b7ff/8sW03MWPEnMcZGSWcPudxHsxU8/LbonrRAqp6u+3btv/4LECNIGbM4/mc5ylylfuyh7nyuoFMoVYXaUkhEpHJTEVm0tT2LW+ToZDq4JFlgrHkEIJqWtXqxpSloK1G7ahsfeICiy8KojC5dXcsGEklZWk8u9wSkD5FqejPjJmRtTzxzlAvXrYWyUW0d//3LA69twWTuuCX2voS664hPoaopkhmyNoubGlFshRnaoX5NCldYmdj2EArHmfinAbms5BPWfDMOoernjOTWeJoXRphQQGHkouOoc9Dn1uoiGg1JuxzhaQ1V7WaX7KOztQAWAU0XMgwW6r6rg9ubL82BFi1JCuWqG+sU5SawEOmIAF96xckuP46W3p2HMc8mzsoixbzPI9DzQhYuqlXIFx7R17pnWBWcQCLIlgNFVBNbWPBZY1QNdH51jXXVQ2m1fwXruk+OvyFXCmsSkxGxmQkEyK6uEBBMR7CVMjjOb8f6i5MMaNK1BqXVpkIE8isqg5ZLD0Rcvm8sjvdW6KNc3txqfMJouswAAiq10pS3xUQzu7ihtIko1sF1r6qOn4IwKik0jyP83w+zzMig61/xzxPsSdUgvTkcLGKMhNEcEJET39aFfSaV7jUpTFVnQQGg5MzGUnorA490D7asUT4Zm6+mW8qYsV69ARgRXCSxKuVJ0H1yMAMVBFBhIBCeTi+HKqMkH2TYTSjWYFEYw9DMsCeBmQdJpdc5dLKq41L59pWfysyM2acUGrWyENVKNWrsVOpaHl6bbE+K5DVHYPde1XbBGUr4qsn23lGpbBU9RImy3lSDGJm1EWsZ5AJkdNOO4+nqNBp7qpmbh7D3V3FgLn43uq409md2iJmZmpuNsZeclpzEymm2URM15oBkyxfWnAmkFClxAogU4SZopquie4Yatib69WFPEDpug4EF/oAqsVUp4sIYJ1JF5FElixagWQggegkSHfep2DlvsuY+SIg8S5AJJBa5k8Xn4SLolkbiJcOQN7iPSk3amZdZhWsCplCeBG0qD78LM57CVkUi2NiBMp3GjTNqve6SYqtnsq6sg9u6q5z6qoUY8UzV1g5mQEEz4ofmanC4QIM1aFGE3fHNgxYsjnRi0cqgQxxVSdeCaimN/gKjIKgFH9qF7sjTdfjCkv0RVABPudcJcI9AVJOuDqHVveDtcH7FrTqbbTRqns1OTF3Hz62sW1j2zdVzeSc5WS5iAasgFBEroTqUpZFIiOjdryJVkGk1sJctXy2pKEwE3fftlHgsXJ4qxCKSUbynOcReczVWCZShdummZvJbQxuzggli7JugfRyoiuQk1XU2txpca5SEVSsHUdQtVrzo0kg6Z/qprtGv3VTAH2ep1wQp4rbReiVv0qqVh3BVY5ReAZCkQ6+zN3G8G23fd/2/Xbbb7d923cRPc+ZGbMypNfjXKn80nhaF1mpqZwAywkUflhHC7v5GNu2uY8+6hcEoCpj2LaPkqYeJ2bmOataCWyZ0Hw8j+/jrN4emaHCbTPyNpz7Te+bVfWJiquYQJuSXCTlC1/X0KMFELWzZzQwZAaFNTBF31V2tstZwD4AqEbfpNo2eRE+uLIKmdVhSQSBMrddzcwsOKVmXmtT1bSOVti2sW2+bWPb94pTt60Q+jkbBibkYhoW3bComPXqcDsny/hpXlbtOkRDLuSaAaGZjc3HdDWtUHbGrDJ0ojrlzOfx/Pp6HOdxnGdGqGS1A/m42Ty3zI3kMj4GSGmh6myM3gKLNZXV9G1ZHkZkXKZOkKqSqzm6dv+O9qJoRTtKNyKK6hf0ZrylGwMvRPduMupGOudQd1QYcQF8H9UhrZLtwsVh/wP/sVDwK2x5izcuVwNFpiyBfETGTGAqq4M3CpdBSj2kot0ZakaS1ToZhQ+6l+uz0NtUISTP21hJ0Ct7+3a3XEh9ZW+kc2mLEbqI2yJvy4N2nxgBDG9nzdQrqxyPVvxIfY+7d612Y6MqM2pJL9s9tMih5A5Sxc0i0q3R3MzNe/xNu9FkJOX3x3u736w8Qg1QlG+86I0ZGSQYlECEzHkUxxdppQQVXamQwvoVt1aup+Q9oFSlfPmDdVhnYRUpPutiDTraKjIeXQZJNpuGC8hfxH7NispqM8b+izLmpWHQbriu3fAZWFnhFMnVvUl8bFsZ98yUSFo267okigVsZW2FSo61MVFRUzG1csC99AWZwZlEJ7nemMkrtRoSAKJPPoiL2it3PZNCQhMxUYLcOZ+HX81npRpICdpMLFVzKeBIUhOQ5ukWMf0W0PaaXpsH0p9WNOvIa+13r11poNIwBEIliXWKMa58lwg6O7U2T/F/eXXJRkWcCaj7GK8JsJVoZwDFg/fYyWJ4rRupaG+H5nvaGhX10TaaYKYAqzrgQm5te9hXewO2BCl5pRNZB0BHZJ7lA6VdTyWx3M1KZ7VoyDZgaO1/5qxDpXDVGBOVYKyItwFYhbrMZMjqdtosvq5EV3vRWkZvRmTBpJdjuxIvEIGC+m6rVvO4mqx0LaaagBj0OghGyJCqbHszY6qVRxJAl3zlLRRYF5VOeMOsunyamo0xhr1eRbd1FldNrOQULpZilcyzdv7tF/ieroaou27btt8ubdllvXs2sbrZXlCx6BBRkC6gqripdzj9z30VcCUYfvtds2N8gWv0Uls/kFcjvXVSwjUBV+m3X7NalIyYkmSdtZkgq43a9X4xs+ogebUOuSxgdwsp5yDVaZLqJEXdtm2vcmAbbuZQJTlFMmecZ8w5E5M4YSGOmVhKkgrx5xrAK3fpbvePHXLfZWPrq7r9Zx3Ademfz3nOecx5nvOcEaqSYUCqwg3Dtc4WkavOax1fwsocZO+Htbau/ZqroVF31mtoXkxFHxZ2pdzfYgu5UmvSFTK9elvWXIyuINB4ZDV7qgBczCjagSY7YGjVMVExeklxumJFxMzHtu23fd82c1d3EU3CzDqWyZiUKTrFU6eck2vhcd1HxDyO+Xicz+OIoLtFfvgQ89YL6zKGBCMlVwRezf3O85jnOTNUNdIBmnIMGUNHNTcX6w3NXnFL6ikQap8GXIJPtEeflUnN9olNMVaf3O7qAvTa51qywldXWK+D0rt7QjlVkbL/FGiu8Gytf3Vr1Vu0WOlCEJFp2T7K3E19uQ41Mx/bvo2xDbPKBFrZQh/DfNjYfKMlPXTghNrlG5IQBBamPI7n19fjPKIa3Hx8bLfbqN5jawi0SP3MiwCpLjkzq6OTQkBTuNfoazc1065AllLQskr2kdnUw2VdWSTH6r3T+6+5AdMqZ6sArFL8y8MtUcaFpuiVvFUAxmuVQwCxJo4yO08urQTW4pSodQpUgZFiqzMsu6a+8ne2JqDCrI64IQIwiKXgj9ez4IrUm1/WlyZSUE4qssRxc3U1A6pHXfEPq1igCfDiV2giNBXFcN9G/dhoNcWL2LnyyADkdR7Aklooqplvvr3QMFiXLEN/iwFqtItP7XmIyyR5Rq7uXFVYXdNQIaFSqSSdtSnXQiBQdIgpDajjXGacnIC55vA3cIg+lSnmCUZE6dAIRPI4zsf3969fv76+vx6P43nO88y5ThupDgVVRVLCsogcYw4/M7IhFzpdmhF5zjzOPM44z1n1wXEywwSbmW0S7oCMbfzx+fHxcbvv+zbczXThUFTnc+lQSRTa4pPXkb4tpUe7d5AqVMgS10nZbKVcs9dWq+U/QU4wSArokVnn6qomrcK5klhaRym48NuCI0j2oTZWPiOSmTPOCaa55jZYp5R0pXv13Z7z1P4egEvs//X9/evnr6/v7+M4z4jMSkC8GNy2Y2Al5s6TEVERSbW3AUFGdm/AYz6PeR6zbM48hemqGIMDlfTf9+3f//j44/PzftvH8E5Rl06YqI7ZFxfKDuk73lZFplzQjO8Nm5q3lDr2o7aPXrJyLnl1zMyTOQsce2axJitpZdUKUGGyBGN1qpmQnHFynoyKNM1crdqOnmfOzJwg4xxR2UqL1XuR3TWMmK0AKxYlns/nNQHnOZMEKpHqWM2crwR3mo+R+zZibgKKyBhWDTkqWxfHGc8jnkfMo0K66mU5zFxdVFTd3Pfb9sfnx8f91hvg4qCQLL6SLQfJ3qwXvBHRa/3wAuyyTqfpkKFQzCtqxiV3aVwwZ+aZGWD6RYC0IG7OMtpScoKaWrOaAGVEV+klYKLSRzDGijVipSdqe15zn8jMmZxFCy8HWWdOPZ+P5/M5ZxBQTYVBISEhUhrzMvoZMzMEaSbV26RKK4sNPud5lsSuqQyIqAlgBhOuRiV1mkWX59hStRdUSZGrDbauvY+OfH6PE7Ly9YVOlgSmrP5bLxF0Df9bU4QyRDljVq9TV+sz1klGTDIYSTOS5s34lH9JcqVf38jyhY5f/dL46uNm2oLJOmSnCrlW/m7RuC8nHBfYJmMCEevIxhlzruE9JzMquwdk5nmexTk/H8fzcZxHBAlRH2Zw1GHpVzvadU4neZH1vTSr1KrVyZkq2q2sZKleV9KkwXnDP1ASUQNvDYWLKSEZPVjNXgoVQOLirNLNnC1UyZgRzLAws4ZOqPJHW2lXXAH1xR+UxRY1ZefIMjlnQCbXAiOxUMnF1rQNvcJbNOXVmczMjDxnAb7znPMZs8RbF6eukJiB5Hwe5/fx/D6P54wzaWKbb1WSV4XtrbtnlPdr7olBtuR5DVPTEQCEfXQv3k6ta/7vUukCVSbATDVxhYlWAzmCkbN4kYJGKlrtq9eRaZkZrmbtXauoIUNmpNty3YAq14FOi1qkSNM4pZhpCMZq/6SZjJi1ZlTZuslOSFXsuKaw+ZZqiwspDyCSZMx4nvE8zhlznseczxJwuNuoUwaquI8xg7Obwq5KEnP3fd9uakpotL2IiJMJ0bmqCl+UUyGgEmymQBRZ51d0vzphyY2y6ygXRyNAz6EJjIsZU62AfUaQLHFBZcOwQGmpTq4z5ckVuQBCUFShUsvnOFOtNnzMeWakKtL6W+RliPpMuMycM0oe6Q6RRm0LA2SvRxAi3UeIsEwhTVSIOeOYcZ5n9Vmf55F5gqkq7mImY5i5iZVvT1Way9h03z0Mrj62LgOkqKFqD7V4XlW533zffNSpRm0kM5OCFNUUE0jqgoFrAlZRUFURN/hJuWKGevtLCp+ZM5OkEtbfsLijZRG9KPvlF7OkPgxgRgIyMx9HVD5SalKgJu6m3T0tpZBl0warh4/ERfkWu5F98Ots3bcCIma67Zv6tt0yq00rkRHP5/F9nuec38/HcRyVxjKrENu3fbvfN3MXZSIjZKWxZfcRZ+kF1DRFIEK9WFsZpjrGdr/5H3/c7/fdx1DVhQ8DKPnUCoaqArfb9a/WhCWt6MC0stbAEp2/Ryad7cgU0YhUlSorB8TURaCkZ8yO+GsmsCioGUEm5/OYj+M8Z0Bk2/x22+73XUTDqaSyz1XlCiOrXQmgIFQ0q7UFsWR1EUwsT2VqY1OIA5od3MR5nhDV78fMeB7H8/kEWb08vY7pue/3j92HSYkQ49yGbm73bRwfeR5xnnnOjJiV/zMt1YDsm+3b2He73fzH5/75cdvGpoacpeOZILmUwSoWIuhsyFU20eZTQDOlULVBf6GsIuvyitMyIwIZUtiCJEKgbgNiJGsHLJJIXkp/EjHjnPH1/fjz1+NxHAK9f9wIjs29TseoBMVqh9BxYF0/UiBpmZm22juvrFe3D1FFlW+47WpOCpgx5/P5nEn905M85zzPqapDxGyMsW97dxnyYa3pDInM+864c555PuPr+/nr+/h+HkyK6DawbbZv+nEf9/u437fbzffbtu/DhwEE57q1BEizK4V7JQUvQXJLqyvYMsFKySwqQkoankuuWd9cUEOEpXUya4fgK9hVM6hdcWr1Rw+SdYLacZwi6nM0+XolKWrhF0WBld17q9Fo87hsXjuEKz6R7sonagotKS6B7Th9eHenLyxcopft9RrDgIiSWKWKA7tk4NimKIMRlAyYyW2z2+63fXzctyLvtt3H5uad4n+7nc4jVu5TrqJW8pV0kcIn7TxKA9FQfFl/Lsh0ocUOz5o0bSGUAC7m1f+s1FTQqvSY8zzP45gaburmbkRxnOav4nxZQwtVV9SJIJXWWrekK4uv2SpPiBSBlKAwzjkjD1Wq1DnLWnZm3zpgyqCpjW3ftn3b9jH2MYpJ69N4QKOkqZoYoMMnkTMnMTM4xvjYb/f77baPfR/77tvm5qX2WPuyCudVpXKEY6vzUIoWUmnftPzuq93HhZ87b3iFGCsMLnH/KupeWksrrAcBXH0XiHoR9pv7IDnP5/PrW8HM3EfebgIbUjLxfffhdVDTWtK191zdUL3wIgF2+tbUzCgwwj2riVhKUplIBjnL9UxT27f9tg81H9vY9/12u91vd4GY+u2277dbz4FvbsOsZOIzC064DRul+I2cxzxLlLdve9XybdsYVclXwUl11UQ3iBORKogYY9u327bfSpsv7EwkRevY+QzOMlZNWRdEvdLznXSt2VARL59yUd1r7dacuPqmIjp822/3j/vYNpDH0yWzGrBFIMV9JlT227bv+1j5iysdWblV0SEQRhKzUNO1DyCidTph1sllLXUM9imdydPNSXHXmuF92+632+f9XtKL7V7NjrdtbG7D1E0khdrH87JEYqWH3DbfN8twEh+37cfn7fPjYxt1evViZSoY5EXot/RojG2MfRs3M4Mkqqi2q/rq2MmK61mFCBWwNa5+JwheTRdLvYA+zK8/0DIFN3cVLXWbjeFjgMyYNoa7j5GRoNICUNl2Xw1mVPXtnKwL2KIiHHk3f1VzyrdXomx0ROZx4jw0M8LSbcx9uDtE3P12u31+fLibqo19VHOhVZBarl2NRrggVS44lgJWK2hAhtsYtnn1ZS0aMwKdwrjcJBpJ2iWHtzqooG5droYawksadLmy7ONDFmUk+bL7y0C9HEi7z3q5b6PyaMX2rH7zFFHzsRGi7jtuFIjY0DF0dIsZWWTldSMgGCXXjoCqSKhNiFIQdeb7nDNm4Jw8gnMmz6MmwGg4/Xg+h1ofLXu77X/824/buQFirqU7GmW+G3epmUEcCCDnOYlznvM4n6X4a1ngqsGtBigApLT83SFkaYTewEJFDoA229mZ4KIzLyBaiCMJFkOskvXR7GLrJUXR7gHMa5xWutXHGMtKICLIA2RE1MgPEdtka4bvKoVnH09zjX2v9AClGbfIVoColl47Zj7PeZznjDN4TJzBGZHzlJia4ZJyHsfz6W3cVG/3m6pkzKbdtWsCVm4NImrq1fNknkcp4M55HsdxnjMyFVYJyXNOyKpa4PtYlgWiAG/kfUO8er5q64HkEiKWDnqFPdn9f9hNc1XqrOgo29VJdRVFoIisAqS9A0rTUYT4jImIypUQqG0BM+nqu7JzUeKKldevyycyKiAutmjOKtIL1VnLJSLPc57nOfMM1s+MYEyJqUwE9TzOpz1FxN1Use9jG9WAe2Z0FyuTi0BGiwQokZwnzjmfz+dxPM/mpGHAjDznPM5JyGiuclnGPv566YCkYXSz/aXmylUFtgKxzpyuFV1vrTnR1cOVq+N8vzVBSahgNX2+Fq+/8ma9XWpsU9DtyToRr4ZmmrFq6ldhDvpIIpHV7aiCMhFSgpRIyhKxFgFZcoNcYsACg5hznsehIsgS+7r57gJGWPdPY6q8up6owk0FIpGnmLxqCLvQH8TUOeecNkVF4YpqrPM6eYbdMaXZnF7rmZXSaq1dVnY5F5H47v1W5UTFBkCd4NEM7NursOnFLvUE5Jz1C5bk/NqAbbGaLawNV9uyyr8re5NRwT5pYtpCMvM+7rBMSdNY5OWhpZXbpmBW+x0hmDHPs8cmXHfddBuuKhk4qvgkV+VlQVyHWU19tjh4G5nzjJlnzpkZKTB3dxvFfrioviUBSsDXYqou6Krc39RUkDnXBNRybmPTFriIh5IgkbUQsRAouo9e2zJe2qnqDlL7wOd59KYopq0cNl9zlpkCROV1BNXNM8nISn0HU9yTQ+Ci6qLmpvCr6+dFtQNCsToSQ1ffL1KyXCOZmed5dC3fNgQcbmKuKcpQpLK83Dr7103MRSUFHNu2xxk5mXGck3nMcwICqA93n2JV+tzQkb3SA+i8ny5kF5mcs3itOOt+2PQzr4w9pIrNKlFQzlwU1CqrK7C5Toyop0eV0lZHvHL4tQNqA17izWoFVaZeqgyEItBK17M6HUfEcZ7zOUlERJ3UYpV5NdFq3wHUGQZoxqK6QUsfQtrZzZCMOgsgSus0J5jzsMxdJE1VTM2Uro2tTVsiU8dBicDCzX2MEWPOoWZs/TpErY7PsszILJZH2gSxKjlq4MtsJsmoAyuQmXFGzkl2F8Ha1AA6Oq4frubNjT27FYsuVXOt5yZVLmZWRERW31BW7VgZniymv6UsEK4Gb+piKUUCz8xzzvN5ZDI8qr7PSUe1N9e+x+LQL7Mp1XkfCk1IdtsiQczZwGFW6nOeI+YZEe4qAndb+KVN5QW1eZFKcpHyFzRbrNSV8MgMhKw/52q+noUkS0/PKAlGRiXPA0Rp/X7PDL+Z+Lff82KGqxdXZ25ElnRFVt86gi5qxaiQyNUlFImuo8w+r4EiYmIwqGmZoJjn8/h+PHKmm8UZeZ7j3McezPBMMSckKpG0iK7ygGoKNcvqPKshUwTCYpAzM+fkcfjj8dh3h3DUeWNjI1jxxJwB1peqKjKyzymbMUswKarmorA+zLabAkcEI0FGlU4RtU3PCJ7HrIM2ZXFakcVUFwXERXsSWEZnCZhoFU5nHdjUC15r0zcpJheSbL8BiosNQZEyWb1jsqSISUmQyD6XWWTokCEOmJZa9jjP5+MRR6jKPM7Yt+08I2ZmbEF1h1rC2vdUS8vqzyam4iWwV4soPRk451TBZEbM43x+P3xsCqF+fIztto0B4HieVZaVkRqSbqKSOY9zPqsK6ZwdxphDdIzNq0eUKcAZs5rRrPUvICPJeQYDFT4W0i3TEgC7EEHafSre6uprT6PsMoSUSBFWF2CoSh16WQI8AJNzxuy8OOjqTlJSEDPJGZl1PFakJJnSmmoRpYnDUpVYEpZ5Huc8pkqJRKsQvvrciHETdQo7KdUs9iLo1LVaKETrgRlpfl7dPuacx/F8PN3ctz1FdWwbRDIpT62ypVI8QCWjTmudz2Mec1a3XDWr7PBiL7QE0zwnW05SYE+aS2CgIk2pBaFIVKkLXkotka6PXGgabJu7TvBlNkRCHVUgXb/S6ffgzCgyhoSrWu3DpEqaSHQKftnvhvzVh7gOVV5W7z0YvlIP9bII0ShqaOWKO8a3JeCuCQAAWjLM1FfLG3SoeFnqliGUqyzejSwBRTdnaQnu7LMq6wPL1TULQ3Z/IS61MVYe++VQ654IgFWxXWxgh6WrBVle8H+BowbtnQFpF6BrR61beMXAdVUXNdTJpCJWWjrVmDMpyUjJYLcSLUDZtHf1Yhk+9k2hAlRHgT6mcrl1ZEKr+RpKu29eWMJE9OpHh04di7n6sIQD2DbfRtkOBXPO4zi08rfmvt/u3tXAOXOW++wip1IZzaIBpOYCy2zXLGUue2IVkdTzV13jKg3rupFh6qru2u2dclGMuTYB3nRZLyppoXqsQLOZwuw9A1GCrloNesVN4aLpEi7HOfmMlGQ00NCaznZQamrDt9ueM8OqpYFYZWzcX3VXNQ09ZepWp2gbYKRmsYZ9Y60/HMOLaNr2rXvbu4F5Hk8yVK3khWZb1gEAzyPOq9C8y//qPLGMNMNcCnJWcV6V7QVFSqSjalAxM7VRoX9Hi6o2dAzbXIeot1S03Ed54GwiqCs0uuisejuJ1eIH0PVaAS1KKZY7Vwq90FEF0NVURWICmkGJ4tMIsFI7l+BXREp5g5T0aGIMoqbDSrSzJMcoDyMqVK0SRyWlOKe13eUyLGN49bbYt9GJR9VkHsfjOMXM9+2+78O3nRQ7DgbPeeBNKZavoqQkkN0Ba/1lZBmparRGIYSOrjQ0k0qHAzCz4WPzffgmYkXsVNH5GxnBJPoY0bb5Yoq1AJtcyqxiI3Q8xtVmlfBC5QC68Xahr2o/7JmAZAqTS+/ODqxUVX3bhJa+iG+KqKyCyc5+YNGGAEr1Jov8Wpu2uiApoE5llipbrJq1lxB5RnCSNHMR83FzyFU30EORL2z/NuKttZZF//b0zGrwVCfUIMK9gLbIKglgreNVHW4hBAOXkW9Lw9JxXb/Qrh9UXRFzu9hiFNAJhT7cR+HvLkEuC18tCIZDgAhQiJQVm8RsIcAYw3VDripFoDLL7m7mADJDMorYwtLGdXMvXu5ZAAW9OrmE9WGgqxCjlvI584hMc4eY2S46VLSEem2KV85nxZ1FDHfxZuHHQJVCl24MghDPRIqaxyCBOjarco1aKLzCosKhS+ukNJhAknXUbtZ5kVcHHXvlXli6epRYBKv98CJEvXMNi1smUpgqYq6ko7JqMWsvMBkzVAiFaaWahtbuaKqvJ0DNwJwx5TxmzFqREUlMUZWSc9Q/IqpGigQyckr3K8rI8yyVM2YcZzwip/lQcbebiJt61Wm1nnb9FGauk76WhMCsoBVQOekZqzlWRJKmvu1bBtjqYyEpgaweLAxZbXML9UqR4sKEClOZIaiyJyuY98rSSxVU9K2B0o60GY23HUAuvWKddqbuZTXQVEKF9ZEhFFdXMfPhm4k3ZF4pqkrZEymnsLpOFY2dSUBKK1m1F6s6sTDhPEtfLWXIQ6pkKc84j3lEnpZpto3xNNvc2uey+3nybQeIqCobeZV+oPjmGo2IVm1KKkTOMYtwK1CZxZAsj5J1wBtXbaMsQqvWT5eUdnV8HVNWSKVNrK5igbq5Tsf0HnD8i5eqgmjBs4pcJcqNAAq3qLm7jvcJ0GsCODOLNpbqOlEPtGivxrQFlet5Fw/Q6yGT1UO1WnBETApm96OJbmvC/BdPILiIr9rzy2/mKxlAIbUbXv3O8veIrX9dfFp/9xUsX399TUBnMF/f9nvy4C83+c8noLRHb7aqERiK1aaseayyHJVVkbxIXVXVyp2UKWwx02oGu0IdrtBEfv8zrvdwnf3bRWJLblb/LW9R0z95iBc7txicDrAWN3fJZctGcw3xu0l7m4BVtN7O9f0K65pcq5t40XDXTPDtVttG/b8trcUaWp/4iwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<PIL.Image.Image image mode=RGB size=128x128 at 0x7F71E8DB6B00>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog\n"
     ]
    }
   ],
   "source": [
    "## parse and confirm data in tfrecords\n",
    "\n",
    "test_index = 21\n",
    "\n",
    "# load and test the tf record files\n",
    "test_image_s3key = 'distributed_training_demo/data/test/test_000.tfrecords'\n",
    "test_set = tf.data.TFRecordDataset(f's3://{bucket}/{test_image_s3key}')\n",
    "\n",
    "# Create a dictionary describing the features.\n",
    "sample_feature_description = {\n",
    "    'image': tf.io.FixedLenFeature([], tf.string),\n",
    "    'label': tf.io.FixedLenFeature([], tf.int64),\n",
    "}\n",
    "\n",
    "def _parse_sample_function(example_proto):\n",
    "    return tf.io.parse_single_example(example_proto, sample_feature_description)\n",
    "\n",
    "test_set = test_set.map(_parse_sample_function)\n",
    "test_set = test_set.batch(1)\n",
    "for i, sample_features in enumerate(test_set):\n",
    "    image = tf.io.decode_raw(sample_features['image'], tf.uint8)\n",
    "    image = tf.reshape(image, (32,32,3))\n",
    "    \n",
    "    label = sample_features['label']\n",
    "    \n",
    "    if i==test_index: break\n",
    "\n",
    "display(Image.fromarray(image.numpy()).resize((128,128)))\n",
    "print(inverted_label_dictionary[str(int(label.numpy()))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog\n"
     ]
    }
   ],
   "source": [
    "## local inference\n",
    "processed_image = tf.reshape(tf.keras.applications.resnet_v2.preprocess_input(image.numpy()), (1,32,32,3))\n",
    "local_vector = model.predict(processed_image)[0]\n",
    "local_prediction = np.argmax(local_vector)\n",
    "print(inverted_label_dictionary[str(int(local_prediction))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dog\n"
     ]
    }
   ],
   "source": [
    "## get inference from batch transform job\n",
    "test_image_s3key = 'distributed_training_demo/batch_transform_output/test_000.tfrecords.out'\n",
    "bt_inference = s3_client.get_object(Bucket=bucket, Key=test_image_s3key)\n",
    "predictions = bt_inference['Body'].read().decode(\"utf-8\")\n",
    "bt_vector = np.array(json.loads(predictions.split()[test_index])['predictions'][0])\n",
    "bt_prediction = np.argmax(bt_vector)\n",
    "print(inverted_label_dictionary[str(int(bt_prediction))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All elements are within relative tolerance of 1e-05\n"
     ]
    }
   ],
   "source": [
    "## check where (if) the endpoint prediction and local prediction disagree\n",
    "rtol = 1e-5\n",
    "not_close_idx = np.where(np.isclose(local_vector, bt_vector, rtol=rtol) == False)[0]\n",
    "for i in not_close_idx:\n",
    "    print(\"Component {} does not agree:\".format(i))\n",
    "    print(\"\\tendpoint_vector: {}\".format(endpoint_vector[i]))\n",
    "    print(\"\\tlocal_vector: {}\".format(local_vector[i]))\n",
    "if not_close_idx.size < 1:\n",
    "    print(\"All elements are within relative tolerance of {}\".format(rtol))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_tensorflow2_p36",
   "language": "python",
   "name": "conda_tensorflow2_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}