{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1be69f0e-bc9c-41c1-ba7b-c5b9a42c2e7e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting sentence-transformers\n",
      "  Using cached sentence_transformers-2.2.2-py3-none-any.whl\n",
      "Collecting transformers<5.0.0,>=4.6.0 (from sentence-transformers)\n",
      "  Downloading transformers-4.31.0-py3-none-any.whl (7.4 MB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.4/7.4 MB\u001b[0m \u001b[31m22.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: tqdm in /opt/conda/lib/python3.10/site-packages (from sentence-transformers) (4.65.0)\n",
      "Requirement already satisfied: torch>=1.6.0 in /opt/conda/lib/python3.10/site-packages (from sentence-transformers) (2.0.0)\n",
      "Requirement already satisfied: torchvision in /opt/conda/lib/python3.10/site-packages (from sentence-transformers) (0.15.1)\n",
      "Requirement already satisfied: numpy in /opt/conda/lib/python3.10/site-packages (from sentence-transformers) (1.23.5)\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/lib/python3.10/site-packages (from sentence-transformers) (1.2.2)\n",
      "Requirement already satisfied: scipy in /opt/conda/lib/python3.10/site-packages (from sentence-transformers) (1.10.1)\n",
      "Collecting nltk (from sentence-transformers)\n",
      "  Using cached nltk-3.8.1-py3-none-any.whl (1.5 MB)\n",
      "Collecting sentencepiece (from sentence-transformers)\n",
      "  Using cached sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "Collecting huggingface-hub>=0.4.0 (from sentence-transformers)\n",
      "  Using cached huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (3.12.0)\n",
      "Requirement already satisfied: fsspec in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2023.5.0)\n",
      "Requirement already satisfied: requests in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (2.28.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (5.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (4.5.0)\n",
      "Requirement already satisfied: packaging>=20.9 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence-transformers) (23.1)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers) (1.11.1)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers) (3.1)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.6.0->sentence-transformers) (3.1.2)\n",
      "Collecting regex!=2019.12.17 (from transformers<5.0.0,>=4.6.0->sentence-transformers)\n",
      "  Using cached regex-2023.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (770 kB)\n",
      "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers)\n",
      "  Using cached tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
      "Collecting safetensors>=0.3.1 (from transformers<5.0.0,>=4.6.0->sentence-transformers)\n",
      "  Using cached safetensors-0.3.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
      "Requirement already satisfied: click in /opt/conda/lib/python3.10/site-packages (from nltk->sentence-transformers) (8.1.3)\n",
      "Requirement already satisfied: joblib in /opt/conda/lib/python3.10/site-packages (from nltk->sentence-transformers) (1.2.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.10/site-packages (from scikit-learn->sentence-transformers) (3.1.0)\n",
      "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /opt/conda/lib/python3.10/site-packages (from torchvision->sentence-transformers) (9.4.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->sentence-transformers) (2.1.2)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (3.1.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence-transformers) (2023.5.7)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.6.0->sentence-transformers) (1.3.0)\n",
      "Installing collected packages: tokenizers, sentencepiece, safetensors, regex, nltk, huggingface-hub, transformers, sentence-transformers\n",
      "Successfully installed huggingface-hub-0.16.4 nltk-3.8.1 regex-2023.6.3 safetensors-0.3.1 sentence-transformers-2.2.2 sentencepiece-0.1.99 tokenizers-0.13.3 transformers-4.31.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install sentence-transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "736dc101-5863-44ff-b086-a25a69f0b51d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /opt/conda/lib/python3.10/site-packages (2.14.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from datasets) (1.23.5)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (12.0.0)\n",
      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.3.6)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from datasets) (2.0.1)\n",
      "Requirement already satisfied: requests>=2.19.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (2.28.2)\n",
      "Requirement already satisfied: tqdm>=4.62.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (4.65.0)\n",
      "Requirement already satisfied: xxhash in /opt/conda/lib/python3.10/site-packages (from datasets) (3.3.0)\n",
      "Requirement already satisfied: multiprocess in /opt/conda/lib/python3.10/site-packages (from datasets) (0.70.14)\n",
      "Requirement already satisfied: fsspec[http]>=2021.11.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (2023.5.0)\n",
      "Requirement already satisfied: aiohttp in /opt/conda/lib/python3.10/site-packages (from datasets) (3.8.5)\n",
      "Requirement already satisfied: huggingface-hub<1.0.0,>=0.14.0 in /opt/conda/lib/python3.10/site-packages (from datasets) (0.16.4)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.10/site-packages (from datasets) (23.1)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /opt/conda/lib/python3.10/site-packages (from datasets) (5.4.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (22.2.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (3.1.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.2)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /opt/conda/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.12.0)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /opt/conda/lib/python3.10/site-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.5.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (3.4)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (1.26.15)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.10/site-packages (from requests>=2.19.0->datasets) (2023.5.7)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/lib/python3.10/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: accelerate in /opt/conda/lib/python3.10/site-packages (0.19.0)\n",
      "Collecting accelerate\n",
      "  Using cached accelerate-0.21.0-py3-none-any.whl (244 kB)\n",
      "Requirement already satisfied: numpy>=1.17 in /opt/conda/lib/python3.10/site-packages (from accelerate) (1.23.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (23.1)\n",
      "Requirement already satisfied: psutil in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.9.5)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/lib/python3.10/site-packages (from accelerate) (5.4.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /opt/conda/lib/python3.10/site-packages (from accelerate) (2.0.0)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.12.0)\n",
      "Requirement already satisfied: typing-extensions in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (4.5.0)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (1.11.1)\n",
      "Requirement already satisfied: networkx in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1)\n",
      "Requirement already satisfied: jinja2 in /opt/conda/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.2)\n",
      "Requirement already satisfied: mpmath>=0.19 in /opt/conda/lib/python3.10/site-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n",
      "Installing collected packages: accelerate\n",
      "  Attempting uninstall: accelerate\n",
      "    Found existing installation: accelerate 0.19.0\n",
      "    Uninstalling accelerate-0.19.0:\n",
      "      Successfully uninstalled accelerate-0.19.0\n",
      "Successfully installed accelerate-0.21.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.1.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.2.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets\n",
    "!pip install  accelerate -U "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b59b9525-038f-4820-9d22-1de0d83bfbf7",
   "metadata": {},
   "source": [
    "# finetune 模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9c2ae124-2a55-4713-afe0-c40be9b3bb5f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sentence_transformers import SentenceTransformer, LoggingHandler\n",
    "from datasets import load_dataset\n",
    "from sentence_transformers import InputExample\n",
    "from torch.utils.data import DataLoader\n",
    "from sentence_transformers import losses\n",
    "import torch\n",
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "07bc65f6-c9a2-47bc-a2da-094b8924ca15",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "logging.basicConfig(format='%(asctime)s - %(message)s', datefmt = '%Y-%m-%d %H:%M:S', level=logging.INFO, handlers =[LoggingHandler()] )"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e2e9e75-5fdf-4e85-967f-e4d86f85902a",
   "metadata": {},
   "source": [
    "## 从hf加载模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "66eade76-c737-4abc-8a2d-625347306f2c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-31 02:19:S - Load pretrained SentenceTransformer: sentence-transformers/paraphrase-multilingual-mpnet-base-v2\n",
      "2023-07-31 02:19:S - Use pytorch device: cuda\n"
     ]
    }
   ],
   "source": [
    "modelB = SentenceTransformer('sentence-transformers/paraphrase-multilingual-mpnet-base-v2')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "4e5cf155-2608-4cad-934f-c7281e3c8fce",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# dataset_id = \"embedding-data/sentence-compression\"\n",
    "# dataset = load_dataset(dataset_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fcfa89a5-e6af-45fa-9b76-7b05900aabbb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['3_faq.faq', '.ipynb_checkpoints', '1_faq.faq', '4_faq.faq', '2_faq.faq']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os,datasets\n",
    "filenames = os.listdir('topwar_faq')\n",
    "filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5808a2dd-569a-4513-9ded-3c25b35a64c9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data size:126\n"
     ]
    }
   ],
   "source": [
    "def parse_faq(file_content,QA_SEP='====='):\n",
    "    arr = file_content.split(QA_SEP)\n",
    "    list_arr = []\n",
    "    for item in arr:\n",
    "        question, answer = item.strip().split(\"\\n\", 1)\n",
    "        question = question.replace(\"Question: \", \"\")\n",
    "        answer = answer.replace(\"Answer: \", \"\")\n",
    "        list_arr.append((answer,question))\n",
    "    return list_arr\n",
    "\n",
    "all_datas = []\n",
    "for fn in filenames:\n",
    "    if fn == '.ipynb_checkpoints':\n",
    "        continue\n",
    "    with open(f\"topwar_faq/{fn}\") as f:\n",
    "        data = f.read()\n",
    "        all_datas += parse_faq(data)\n",
    "print(f\"data size:{len(all_datas)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "734c79a3-92b1-4740-909e-e7e195099bf6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_examples = []\n",
    "\n",
    "for i in range(len(all_datas)):\n",
    "    example = all_datas[i]\n",
    "    train_examples.append(InputExample(texts=[example[0], example[1]],label = [0.85]*len(all_datas)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "95c874f8-1f0c-40d9-b9b3-6d4a219646fe",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "train_dataloader = DataLoader(train_examples, shuffle=True, batch_size=64)\n",
    "train_loss = losses.MultipleNegativesRankingLoss(model=modelB)\n",
    "num_epochs = 10\n",
    "warmup_steps = int(len(train_dataloader) * num_epochs * 0.1) #10% of train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c287779-5147-4d07-93d3-e595b674914f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# from sentence_transformers.evaluation import EmbeddingSimilarityEvaluator\n",
    "\n",
    "# # evaluator = EmbeddingSimilarityEvaluator.from_input_examples(\n",
    "# #     train_examples\n",
    "# # )\n",
    "# sentences1 = ['世界上最好玩的游戏，江娱互动旗下的一款SLG游戏', '专属技能碎片可以通过多种途径获得，例如礼包商城-特惠礼包界面可以购买专属技能碎片礼包']\n",
    "# sentences2 = ['口袋奇兵是什么？', '专属技能碎片在哪里获得？']\n",
    "# scores = [0.75, 0.8]\n",
    "\n",
    "# evaluator = EmbeddingSimilarityEvaluator(sentences1, sentences2, scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7353170e-a8af-494b-8981-40f586446955",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch:   0%|          | 0/10 [00:00<?, ?it/s]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:03<00:03,  3.73s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:05<00:00,  2.57s/it]\u001b[A\n",
      "Epoch:  10%|█         | 1/10 [00:05<00:46,  5.15s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.44s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.42s/it]\u001b[A\n",
      "Epoch:  20%|██        | 2/10 [00:08<00:30,  3.80s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.44s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.41s/it]\u001b[A\n",
      "Epoch:  30%|███       | 3/10 [00:10<00:23,  3.36s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.44s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.42s/it]\u001b[A\n",
      "Epoch:  40%|████      | 4/10 [00:13<00:18,  3.15s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.45s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.42s/it]\u001b[A\n",
      "Epoch:  50%|█████     | 5/10 [00:16<00:15,  3.04s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.43s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.43s/it]\u001b[A\n",
      "Epoch:  60%|██████    | 6/10 [00:19<00:11,  2.98s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.42s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.42s/it]\u001b[A\n",
      "Epoch:  70%|███████   | 7/10 [00:22<00:08,  2.94s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.47s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.43s/it]\u001b[A\n",
      "Epoch:  80%|████████  | 8/10 [00:25<00:05,  2.92s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.45s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.44s/it]\u001b[A\n",
      "Epoch:  90%|█████████ | 9/10 [00:27<00:02,  2.91s/it]\n",
      "Iteration:   0%|          | 0/2 [00:00<?, ?it/s]\u001b[A\n",
      "Iteration:  50%|█████     | 1/2 [00:01<00:01,  1.47s/it]\u001b[A\n",
      "Iteration: 100%|██████████| 2/2 [00:02<00:00,  1.44s/it]\u001b[A\n",
      "Epoch: 100%|██████████| 10/10 [00:30<00:00,  3.09s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-31 02:55:S - Save model to ./finetuned-sentence-embedding\n"
     ]
    }
   ],
   "source": [
    "torch.cuda.empty_cache()\n",
    "modelB.fit(train_objectives=[(train_dataloader, train_loss)],\n",
    "          epochs=num_epochs,\n",
    "           # evaluator=evaluator,\n",
    "            evaluation_steps=5,\n",
    "           output_path='./finetuned-sentence-embedding',\n",
    "          warmup_steps=warmup_steps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "dc41acd2-7471-4cc1-889b-1a70eb6d21c9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# modelB.evaluate(evaluator)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4597bff-501a-47e0-83df-a4a672823f18",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 从本地加载finetuned的模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "18fd2552-4321-4c94-8452-2177b731554b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-31 03:02:S - Load pretrained SentenceTransformer: ./finetuned-sentence-embedding\n",
      "2023-07-31 03:03:S - Use pytorch device: cuda\n"
     ]
    }
   ],
   "source": [
    "modelB = SentenceTransformer('./finetuned-sentence-embedding')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "82f99adb-d475-41bc-827b-9fe1dbdd96d8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 1/1 [00:00<00:00, 60.77it/s]\n"
     ]
    }
   ],
   "source": [
    "input_sentences_a = ['专属技能碎片可以通过多种途径获得，例如礼包商城-特惠礼包界面可以购买专属技能碎片礼包', \n",
    "                  '中国首都在北京',\n",
    "                  '美国首都在华盛顿']\n",
    "input_sentences_q = ['专属技能碎片在哪里？',\n",
    "                   '中国首都在哪里？',\n",
    "                   '美国首都在哪里？']\n",
    "\n",
    "embeddings = modelB.encode(input_sentences_a+input_sentences_q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "cb77deba-a613-4177-bf04-7f02692614d5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.7496, -0.0593, -0.0969],\n",
       "        [-0.0216,  0.8824,  0.5007],\n",
       "        [-0.0075,  0.5930,  0.8479]])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sentence_transformers import evaluation,util\n",
    "\n",
    "util.cos_sim(embeddings[:3],embeddings[3:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "807f79e8-002a-4f86-9294-27b11158cbb3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "input_answer  = []\n",
    "input_question  = []\n",
    "for i, (a,b) in enumerate (all_datas):\n",
    "        input_answer.append(a)\n",
    "        input_question.append(b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "a9bbeb9d-7d2d-4e47-aa4c-3cacb31db498",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Batches: 100%|██████████| 4/4 [00:00<00:00,  8.50it/s]\n",
      "Batches: 100%|██████████| 4/4 [00:00<00:00, 31.24it/s]\n"
     ]
    }
   ],
   "source": [
    "emb_answer = modelB.encode(input_answer)\n",
    "emb_question = modelB.encode(input_question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "90bb0891-fd80-4a90-bdf1-097c43c80e55",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(126, 126)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(input_question),len(input_answer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ffe883b-4768-4236-a55a-0a71fc921516",
   "metadata": {},
   "source": [
    "## 查看sim得分分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "cb522ef6-03c0-4922-b307-2ffbb3ff9727",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cos = torch.nn.CosineSimilarity()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "6f71a4c1-3545-4a43-bb00-01ef1e2c4d6d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "simsvalues = cos(torch.tensor(emb_answer),torch.tensor(emb_question)).numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "d6e8ed13-507b-422d-8678-7cbf5463a062",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-31 03:16:S - generated new fontManager\n"
     ]
    }
   ],
   "source": [
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "c4c3982e-0820-4fdf-8964-6bcd2ec4d3f9",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGdCAYAAADnrPLBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABMtUlEQVR4nO3dd3hUVeI+8PdOyWTSJo00UmhJCKF3EKWjqBSx4KIsrGV1BZVl1RVZFX+rYtlFVMBdXVEsiLsqykozUgJIDwQIJBBCwoSQNmmTZJJJMnN/fwTy3UhNmMmZO3k/z3Ofx6l5c4mTN/eee44ky7IMIiIiIoVSiQ5AREREdCNYZoiIiEjRWGaIiIhI0VhmiIiISNFYZoiIiEjRWGaIiIhI0VhmiIiISNFYZoiIiEjRNKIDOJvdbsf58+fh6+sLSZJExyEiIqLrIMsyKisrERERAZXq6sde3L7MnD9/HlFRUaJjEBERUSvk5uYiMjLyqs9x+zLj6+sLoHFn+Pn5CU5DRERE18NsNiMqKqrp9/jVuH2ZuXhqyc/Pj2WGiIhIYa5niAgHABMREZGiscwQERGRorHMEBERkaKxzBAREZGiscwQERGRorHMEBERkaKxzBAREZGiscwQERGRorHMEBERkaKxzBAREZGiscwQERGRorHMEBERkaKxzBAREZGiuf2q2UQkjtFohMlkEh3jhgUHByM6Olp0DCK6ApYZInIKo9GI7gkJqLFYREe5YXovL2Skp7PQELkolhkicgqTyYQaiwUP/PlthEZ3FR2n1QqNWfjyzWdhMplYZohcFMsMETlVaHRXRMYmio5BRG6MA4CJiIhI0VhmiIiISNFYZoiIiEjRWGaIiIhI0VhmiIiISNFYZoiIiEjRWGaIiIhI0VhmiIiISNFYZoiIiEjRWGaIiIhI0VhmiIiISNFYZoiIiEjRWGaIiIhI0VhmiIiISNFYZoiIiEjRhJaZDz74AL1794afnx/8/PwwbNgwbNy4selxWZaxaNEiREREQK/XY9SoUTh+/LjAxERERORqhJaZyMhIvPHGGzh48CAOHjyIMWPGYMqUKU2F5a233sKSJUuwbNkyHDhwAGFhYRg/fjwqKytFxiYiIiIXIrTMTJo0Cbfffjvi4uIQFxeH1157DT4+Pti7dy9kWcbSpUuxcOFCTJs2DT179sSqVatgsViwevVqkbGJiIjIhbjMmBmbzYY1a9aguroaw4YNQ3Z2NgoKCjBhwoSm5+h0OowcORK7d+++4vtYrVaYzeZmGxEREbkv4WXm2LFj8PHxgU6nw+OPP461a9eiR48eKCgoAACEhoY2e35oaGjTY5ezePFiGAyGpi0qKsqp+YmIiEgs4WUmPj4eqamp2Lt3L/7whz9g1qxZOHHiRNPjkiQ1e74sy5fc978WLFiAioqKpi03N9dp2YmIiEg8jegAHh4e6NatGwBg4MCBOHDgAN599138+c9/BgAUFBQgPDy86flFRUWXHK35XzqdDjqdzrmhiYiIyGUIPzLza7Isw2q1onPnzggLC0NSUlLTY3V1dUhOTsbw4cMFJiQiIiJXIvTIzAsvvICJEyciKioKlZWVWLNmDbZv345NmzZBkiTMmzcPr7/+OmJjYxEbG4vXX38dXl5emDFjhsjYRERE5EKElpnCwkLMnDkT+fn5MBgM6N27NzZt2oTx48cDAJ577jnU1NTgiSeeQFlZGYYMGYKffvoJvr6+ImMTERGRCxFaZj7++OOrPi5JEhYtWoRFixa1TSAiIiJSHJcbM0NERETUEiwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGgsM0RERKRoLDNERESkaCwzREREpGhCy8zixYsxaNAg+Pr6IiQkBFOnTsXJkyebPWf27NmQJKnZNnToUEGJiYiIyNUILTPJycmYM2cO9u7di6SkJDQ0NGDChAmorq5u9rzbbrsN+fn5TduGDRsEJSYiIiJXoxH5xTdt2tTs9ieffIKQkBCkpKTglltuabpfp9MhLCysreMRkQA2uwxzTT2sDXbYZBkalQQ/vRaeGhUkSRIdj4hckNAy82sVFRUAgMDAwGb3b9++HSEhIfD398fIkSPx2muvISQk5LLvYbVaYbVam26bzWbnBSaiG2azyzhXZkGOyQJjmQXlljrY5Uufp9OoEBmgR0ygN7qF+EDvoW77sETkklymzMiyjPnz52PEiBHo2bNn0/0TJ07Evffei5iYGGRnZ+PFF1/EmDFjkJKSAp1Od8n7LF68GK+88kpbRieiVqi2NuBYXgWO5VXAUmdr9phWLcFTq4ZKklBvs8NSZ4O1wY6s4mpkFVcjObMY3cN80S/KH0E+l34OEFH74jJlZu7cuTh69Ch27drV7P7p06c3/XfPnj0xcOBAxMTEYP369Zg2bdol77NgwQLMnz+/6bbZbEZUVJTzghNRi9TW23DwbBlSc8thu3AIRq9Vo0sHb3QK8kaonw4+Ok2zU0oNNjtMVXUwllpwurgKxZVWHD9vxol8M3p3NGBolyB4anmkhqi9coky8+STT2LdunXYsWMHIiMjr/rc8PBwxMTEIDMz87KP63S6yx6xISKxZFnG8XwzdmWaYG2wAwDC/DzRL9ofXTv4QK268ngYjVqFMIMnwgyeGNQpAPkVtUg5W4YzpmocOVeBU4VVmNAjFJ2Cvdvq2yEiFyK0zMiyjCeffBJr167F9u3b0blz52u+pqSkBLm5uQgPD2+DhETkCBU19fg5vRDnymoAAEHeHripWzA6BXm1eFCvJEmI8Ncjwl8PY6kFySeLUWqpww9HzmNATACGdQm6ajEiIvcjtMzMmTMHq1evxg8//ABfX18UFBQAAAwGA/R6PaqqqrBo0SLcfffdCA8PR05ODl544QUEBwfjrrvuEhmdiK7TOYuEw/uNqGuwQ6OSMKxLEPpG+UPlgMIRHeiF3wyOwq7TJhw5V4GUs2UwVVlxe89weGg4JyhReyH0//YPPvgAFRUVGDVqFMLDw5u2r7/+GgCgVqtx7NgxTJkyBXFxcZg1axbi4uKwZ88e+Pr6ioxORNfQYJcRMO5x7DNpUddgR7jBEw8MiUb/mACHFJmLNGoVRsWH4PZeYdCoJJwtseDbQ+dQbW1w2NcgItcm/DTT1ej1emzevLmN0hCRo5RW1+H/JZfCb8CdAICBMQEY6uTTP7EhvvDVabHuyHkUVVrx3aE83D2gI7w8XGJoIBE5EY/DEpFDnSmuwtTlvyCtuA52qwXDgutxU7fgNhnHEmbwxL0DI+Gj06DUUoe1h/NQW2+79guJSNFYZojIYQ4by3DPP/bAWGpBqLcaBV88gwivqx+BdbQALw9M698RXh5qmKrq8EPqedTb7G2agYjaFssMETnErkwTZny0D6XVdegdacDisUGoNxmFZAnw8sBd/TrCU6NCgbkWP50ovOZpbSJSLpYZIrph208W4aFVB1BTb8MtcR3w1aND4e8pdhK7YB8d7ugdDpUEnC6qwt4zpULzEJHzsMwQ0Q3Zkl6I33+WgroGO8b3CMVHvx0Ab51rDLqNDPDCmO6N67jtzylFZmGl4ERE5AwsM0TUapuPF+DxL1JQZ7PjtsQwLJ/RHzqNay0rkBhhQP9ofwDAz+lFKLfUiQ1ERA7HMkNErbIpLR9zvjyEepuMO3qH4/0Z/Vx2orrhXYMRYfBEnc2ODccK0MABwURuxTU/eYjIpW3LKMLc1YfRYJcxpW8E3p3eF1q1636cqFUSJvYMh16rRnGVFTtPm0RHIiIHct1PHyJySfvOlODxL1LQYJcxuU8EltzXFxoXLjIX+XhqcGtiKADg6LkKnC2pFpyIiBzF9T+BiMhlHD1XjodXHYS1wY6x3UPw9/v6KGpRx5ggb/SJNAAAktILOaEekZtgmSGi65JZWIlZK/ejytqAYV2CsPyB/i59aulKbuoWjAAvLaqtNmw/WSw6DhE5gPI+iYiozeWWWvDAv/ahzFKPPpEGfDRrIDy1rnXV0vXSqlWYkBgGSQJOFlYi28TTTURKxzJDRFdVWl2H367cj6JKK+JDffHp7wbDx0XmkWmtMD9P9I8KAABszShCXQOvbiJSMpYZIroiS10DHvr0ALJN1ejor8eqhwYjwNtDdCyHGNIlEAa9FlXWBuzO4tVNRErGMkNEl9Vgs2Pu6sNIzS2HQa/FqocGIczgKTqWw2jVqqbZgY+cq0ChuVZwIiJqLZYZIrqELMt4Ye0xbM0ogk6jwsrZA9EtxFd0LIeLDvRCfFjj95V8qpiLURIpFMsMEV3inaRT+PfBc1BJwLIZ/TEgJlB0JKcZ0TUYWrWE/IpanCzg2k1ESsQyQ0TNfLH3LN7behoA8OrUXhjfI1RwIufy8dRgUKfGsrbrtImDgYkUiGWGiJpsPl6Al35IAwA8NTYWM4ZEC07UNvpF+8Og16K6zob9OaWi4xBRC7HMEBEA4GBOKZ766jDsMnD/oCj8cVys6EhtRqNS4Za4YADAYWMZyriyNpGisMwQETILK5uWKRiXEIJXp/aEJClnmQJH6BzkjZggL9hlYMcpzgxMpCQsM0TtXEFFLWat3I+Kmnr0i/bH+7/pr4iFIx1NkiSMjO0AlQTklFg4MzCRgrS/TywialJhqceslftxvqIWXYK98fGsQdB7KHOZAkcI8PZA3yh/AMAvp02w81JtIkVgmSFqp2rrbXj0s4M4WViJEF8dVj00GIFuMrvvjRjcKRA6jQol1XXIyOel2kRKwDJD1A7Z7DLmrUnF/pxS+Oo0+PR3gxEV6CU6lkvQadVNl2rvOVMCGw/OELk8lhmidkaWZbz0Qxo2HS+Ah1qFD387ED0i/ETHcil9Ig3w0WlQZW1AViU/JolcHf8vJWpn3t96Gl/uM0KSgKX398WwrkGiI7kcjVqFoV0aj85kmNWQdN6CExHR1bDMELUjX+03YknSKQDAK5MTcXuvcMGJXFdCuB8CvT1Qb5dgGHK36DhEdBUsM0TtRNKJQixcewwAMHd0N/x2WCexgVycSpJw04WjVr4DJ6O0xiY4ERFdCcsMUTtwMKcUc1cfgl0G7hsYiT9NiBMdSRE6B3sjSGeHSuuJr49XiY5DRFfAMkPk5k79z+y+Y7uH4PW7erW72X1bS5Ik9DQ0HpHZks2J9IhcFcsMkRvLr6hpmt23f7Q/ls1on7P73ohgTxmW0/thl4H3t2SKjkNEl8FPNSI3VVJlxYP/2of8ilp0C/Fp97P73oiKXV8CAL5PzcPpIp5uInI1LDNEbshcW49Zn+xHVnE1IgyeWPXQYARwdt9WqyvMwqAIHewy8B6PzhC5HJYZIjdTU2fDI58eRFqeGUHeHvjikSHo6K8XHUvxpif6AgD+e/Q8ThVymQMiV8IyQ+RG6hrs+MOXKY3LFHhq8NnDg9Glg4/oWG6hS4AWtyWGQZaBd3l0hsilaEQHIKJLGY1GmEymFr3GZpfxzr5y7M6thYcaeH64AdaCLBwqcFLIa0hPTxfzhZ1o3vhYbDpegPVH8/HkGDO6h3EZCCJXwDJD5GKMRiO6JySgxmJp0esCb3sSvn1uhWyrR+6//4oHXz/kpIQtU1XlPgNmu4f54Y5e4Vh/LB9LkzLxj5kDREciIrDMELkck8mEGosFD/z5bYRGd73m82UZOFauRmalGoCMoaFA5DN/cX7Qa0jfn4yNq95FbW2t6CgO9fS4WGxIy8em4wU4fr4CiREG0ZGI2j2WGSIXFRrdFZGxidd83t4zJcisLAUAjEsIdZlfroXGLNERnCIu1BeTekdg3ZHzWPpzJj767UDRkYjaPQ4AJlKw/dml2JfdWGRuiQ12mSLj7p4aGwuV1Lje1bFzFaLjELV7LDNECnXwbCn2nCkBAIzoFox+0QGCE7Uf3UJ8MKVvRwDAOz+fEpyGiFhmiBTokLEMv5xuLDLDugRhQAyLTFt7amws1CoJWzOKcNhYJjoOUbvGMkOkMEdyy7Ezs/Gy7cGdAzG4c6DgRO1T52Bv3NXv4tEZzjtDJJLQMrN48WIMGjQIvr6+CAkJwdSpU3Hy5Mlmz5FlGYsWLUJERAT0ej1GjRqF48ePC0pMJNaxvApsP1UMABgYE4ChLDJCPTUmFhqVhB2nipFylkdniEQRWmaSk5MxZ84c7N27F0lJSWhoaMCECRNQXV3d9Jy33noLS5YswbJly3DgwAGEhYVh/PjxqKzkdOLUvhw/X4GtGUUAgP7R/hjeNQiSJAlO1b5FB3nh7v6RAIClHDtDJIzQMrNp0ybMnj0biYmJ6NOnDz755BMYjUakpKQAaDwqs3TpUixcuBDTpk1Dz549sWrVKlgsFqxevVpkdKI2lZ5vxs/pjUWmb6Q/RnQLZpFxEXPHdINGJWFnpgkHckpFxyFql1xqzExFReMljoGBjYfOs7OzUVBQgAkTJjQ9R6fTYeTIkdi9e/dl38NqtcJsNjfbiJTsZEElkk4UAgB6dTTgljgWGVcSFeiFewdGAQDeSeLRGSIRXKbMyLKM+fPnY8SIEejZsycAoKCgcVGZ0NDQZs8NDQ1teuzXFi9eDIPB0LRFRUU5NziRE50qrMTmEwWQASRG+GF0fAcWGRc0d0w3aNUSdmeVYO+Fy+WJqO24TJmZO3cujh49iq+++uqSx3794S3L8hU/0BcsWICKioqmLTc31yl5iZzNWK3CprQCyDKQEO6Lsd1DWGRcVEd/Pe7j0RkiYVyizDz55JNYt24dtm3bhsjIyKb7w8LCAOCSozBFRUWXHK25SKfTwc/Pr9lGpDTeiaNxoEQNGUCPcD+MSwhlkXFxc0Z3g4dahX3Zpdid1bIVz4noxggtM7IsY+7cufjuu++wdetWdO7cudnjnTt3RlhYGJKSkpruq6urQ3JyMoYPH97WcYnaxJZsC4Lu+CMACYkRfhiXEAIVi4zLi/DX4/7BjUdnliZlQpZlwYmI2g+hZWbOnDn44osvsHr1avj6+qKgoAAFBQWoqakB0Hh6ad68eXj99dexdu1apKWlYfbs2fDy8sKMGTNERidyiq/2G7H8QAUkSYUuPjaeWlKYJ0Z1g4dGhf05pU0zNBOR8wktMx988AEqKiowatQohIeHN21ff/1103Oee+45zJs3D0888QQGDhyIvLw8/PTTT/D19RWYnMjxvth7Fgu+OwYAMB9ch74BNhYZhQkzeGLG4GgAjWs28egMUdsQfprpctvs2bObniNJEhYtWoT8/HzU1tYiOTm56WonInexancO/vJ9GgBgUpw3yrZ8CPYYZXpiVFfoNCqknC1rWnaCiJzLJQYAE7VnH+/KxsvrGpfoeOyWLpjdh0cdlSzEzxMPDo0BACxJ4tEZorbAMkMk0Ic7svDXH08AaPyL/vmJ3XlqyQ08PrIrPLUqpOaWN62lRUTOwzJDJMiK7afx+oYMAMBTY2Px7K3xLDJuooOvDjMvHJ1ZyqMzRE7HMkMkwHtbMvHWpsYV4v84Lg7zx8exyLiZx0Z2hV6rxpFz/7dAKBE5B8sMURuSZRlLfjqJJRdmiX321ng8PS5WcCpyhmAfHX47/MLRmZ857wyRM7HMELURWZbx6vp0vLf1NABgwcTumDO6m+BU5EyP3dIVXh5qHMuraFr1nIgcj2WGqA3Y7TJeWJuGj3dlAwAWTeqBx0Z2FZyKnC3Q2wOzh3cC0Lhmk93OozNEzsAyQ+RkDTY7/vSfI/hqvxEqCXjr7t6YfVPna7+Q3MKjN3eBj06DE/lm/HgsX3QcIrfEMkPkRNYGG+asPoS1h/OgUUl49/5+uG9QlOhY1IYCvD3w+1u6AAD+/tNJ1DXYBScicj8sM0ROUlNnw+8/S8Hm44XwUKvwjwcHYFKfCNGxSICHR3RGsI8OZ0ss+PqAUXQcIrfTqjLTpUsXlJRcuohaeXk5unTpcsOhiJSuytqA2Z/sR/KpYui1aqycPQjjeoSKjkWCeOs0eGps42Dvd7echqWuQXAiIvfSqjKTk5MDm812yf1WqxV5eXk3HIpIySos9XjgX/uwL7sUvjoNPnt4MEbEBouORYLdPyga0YFeMFVZsfLCQHAicgxNS568bt26pv/evHkzDAZD022bzYYtW7agU6dODgtHpDSmKitmfrwf6flm+Htp8flDQ9Ar0nDtF5Lb89Co8KcJcXh6TSr+mXwGDwyJQYC3h+hYRG6hRWVm6tSpABpXsp41a1azx7RaLTp16oS///3vDgtHpCTnyiz47cf7ccZUjQ6+Onzx8BDEh3HRSPo/k3pH4J/JZ3Ai34zl207jL3f2EB2JyC206DST3W6H3W5HdHQ0ioqKmm7b7XZYrVacPHkSd955p7OyErmszMJK3PPBHpwxVaOjvx7/fmwYiwxdQqWS8Nxt8QCAz/acRV55jeBERO6hVWNmsrOzERzMMQBEAHDYWIZ7/7kHBeZaxIb44Js/DEPnYG/RschFjYzrgKFdAlFns+Nvm0+KjkPkFlp0mul/bdmyBVu2bGk6QvO/Vq5cecPBiJRgx6liPP5FCix1NvSN8scnswdxHARdlSRJeOH2BExe9gvWHs7Db4fFoF90gOhYRIrWqiMzr7zyCiZMmIAtW7bAZDKhrKys2UbUHvx49DweXnUAljobbo4NxpePDGGRoevSO9Ifd/ePBAD8vx9PcBFKohvUqiMz//jHP/Dpp59i5syZjs5DpAif7z2Ll35IgywDd/YOx5L7+sJDwzko6fo9d1s8Nqbl47CxHD+knsfUfh1FRyJSrFZ9+tbV1WH48OGOzkLk8mRZxjtJp/Di941F5sGh0Xj3/n4sMtRioX6eTaumv7ExgxPpEd2AVn0CP/LII1i9erWjsxC5tHqbHc99cxTvbskEADw1phv+OqUn1CpJcDJSqodHdEZHfz0KzLX4Z/IZ0XGIFKtVp5lqa2vx4Ycf4ueff0bv3r2h1WqbPb5kyRKHhCNyFVXWBjzx5SHsOFUMlQS8OrUXZgyJFh2LFM5Tq8YLtydgzupD+OeOLEwfFIUIf73oWESK06oyc/ToUfTt2xcAkJaW1uwxSeJfqeReisy1mP3JAZzIN0OvVWPZjH4Ym8B1lsgxbu8VhsGdArE/pxRvbsrAu/f3Ex2JSHFaVWa2bdvm6BxELimzsBKzPzmAvPIaBPt44ONZg9Anyl90LHIjkiThpUk9MGnZLvyQeh73D4rGsK5BomMRKQpHLRJdwd4zJbj7g93IK69Bl2BvfPeHm1hkyCl6djTggQunLReuPYba+ksX8iWiK2vVkZnRo0df9XTS1q1bWx2I6EYYjUaYTKYbfp/ksxYsP1CBBjsQH6TFCzf5wHQ2A6azDgh5Denp6c7/IuRynr21OzYfL8QZUzVWbM/C/PFxoiMRKUaryszF8TIX1dfXIzU1FWlpaZcsQEnUVoxGI7onJKDGYrmBd5Hgf8tMGIbdBwCoPvkLkn78O35qqHNMyBaoqqpq869J4hj0WiyalIg5qw/hg+2nMblPOLqFcH0vouvRqjLzzjvvXPb+RYsW8QOYhDGZTKixWPDAn99GaHTXFr++wQ4cKNHgfE3j2dd4PxsSxw2CNH6No6NeVfr+ZGxc9S5qa2vb9OuSeLf3CsOY7iHYmlGEF75Lw5rfD4WKl/4TXVOr12a6nAcffBCDBw/G3/72N0e+LVGLhEZ3RWRsYoteY66tx3+PnIeppg5qScLYhBAkhPs5KeHVFRqzhHxdEk+SJLwyORF7skqwP6cU/0nJxfRBnAKA6FocOgB4z5498PT0dORbEjldfkUNvj6QC1NVHfRaNe4e0FFYkSGKCvRqGi/z+oYMmKqsghMRub5WHZmZNm1as9uyLCM/Px8HDx7Eiy++6JBgRG0ho8CMn9OLYLPLCPbxwKTeEfDTa6/9QiIn+t1NnbD2cB5O5Jvx4vdpWPFAf87hRXQVrToyYzAYmm2BgYEYNWoUNmzYgJdfftnRGYkczmaXseNUMTYfL4TNLqNLsDfuHRDFIkMuQaNW4a17ekOjkrAxrQDfHsoTHYnIpbXqyMwnn3zi6BxEbaba2oCNaQXIK68BAAyMCcDwrkH8y5dcSs+OBvxxfBze3nwSi9Ydx5DOgYgK9BIdi8gl3dAA4JSUFKSnp0OSJPTo0QP9+nEabnJtBRW1WH8sH1XWBmjVEib0CEO3EB/RsYgu6/GRXbEtowgHz5bhj1+n4uvHhnFhU6LLaFWZKSoqwv3334/t27fD398fsiyjoqICo0ePxpo1a9ChQwdH5yS6IbIsI+28Gckni2GTZQR4aXFn7wgEenuIjkZ0RWqVhHem98XEd3fi4Nky/CM5C3NGdxMdi8jltGrMzJNPPgmz2Yzjx4+jtLQUZWVlSEtLg9lsxlNPPeXojEQ3pMFmx8/pRdiaUQSbLKNrB29MHxTFIkOKEBXohUWTG6caeCfpFI6eKxcbiMgFterIzKZNm/Dzzz8jISGh6b4ePXpg+fLlmDBhgsPCUdtx1DIAIl1uGQBzbT3WH81HUaUVEoDhXYMwICaA42NIUe7u3xFbMwqx4VgB5n2div/OHQFvnUOnCSNStFb932C326HVXnrVh1arhd1uv+FQ1LYcswyA67g4C7Wx1IJNaQWoqbfBU6PCbT3DEBPkLTgdUctJkoTXpvZCytkynCmuxoLvjuHd+/uylBNd0KoyM2bMGDz99NP46quvEBERAQDIy8vDH//4R4wdO9ahAcn5bnQZAFdxcRmAmppaHDxbit2nSyADCPHV4Y5e4bzsmhQtwNsDy2b0x/0f7sW6I+cxsFMAfjusk+hYRC6hVWVm2bJlmDJlCjp16oSoqChIkgSj0YhevXrhiy++cHRGaiOtWQbAlRQasyB56HGiLgCm0yUAgIRwX4yJD4FG7dDJromEGNQpEAsmdser69Px1x9PIDHCgAExAaJjEQnXqjITFRWFQ4cOISkpCRkZGZBlGT169MC4ceMcnY/oulnggbCZf4fJpodKAkbGdUCvjgYeiie38vCIzjhkLMOGYwV47PMU/DD3JnT014uORSRUi/5c3bp1K3r06AGz2QwAGD9+PJ588kk89dRTGDRoEBITE7Fz506nBCW6mqziKhxBJ3gER8NDsuGeAZHoHenPIkNuR5IkvH1PH3QP84WpyopHVx2Epa5BdCwioVpUZpYuXYpHH30Ufn6XLsJnMBjw2GOPYcmSJQ4LR3Qtsixj75kS/Hg0HzaoUWs8hgGexQg38C9Vcl/eOg3+NWsggrw9cCLfjKfXpMJml0XHIhKmRWXmyJEjuO222674+IQJE5CSknLDoYiuh7XBhh+P5mNfdikAIBylKPz6L/CQeEUdub/IAC/8c+YAeGhUSDpRiEXrjkOWWWiofWpRmSksLLzsJdkXaTQaFBcXX/f77dixA5MmTUJERAQkScL333/f7PHZs2dDkqRm29ChQ1sSmdxUmaUO/z5wDmdM1VCrJIxPCEVXFAJ2m+hoRG1mYKdALJ3eF5IEfL73LFZszxIdiUiIFpWZjh074tixY1d8/OjRowgPD7/u96uurkafPn2wbNmyKz7ntttuQ35+ftO2YcOGlkQmN5RjqsaaA7kotdTBW6fGPf0j0SPi0lOfRO3B7b3C8eIdPQAAb28+ic/35IgNRCRAi65muv322/HSSy9h4sSJ8PT0bPZYTU0NXn75Zdx5553X/X4TJ07ExIkTr/ocnU6HsLCwlsQkNyXLMg6eLcPurMbLrsMNnrijVzhnQqV276ERnVFaXYdl207jxR+OQ++hwT0DIkXHImozLfot8Je//AXfffcd4uLiMHfuXMTHx0OSJKSnp2P58uWw2WxYuHChQwNu374dISEh8Pf3x8iRI/Haa68hJCTkis+3Wq2wWq1Nty9eeUXK1mC3Y0t6ETIKKgEAPSP8MCo+hCsIE13wpwlxqK5rwCe/5OC5b45AAnA3Cw21Ey0qM6Ghodi9ezf+8Ic/YMGCBU2DzSRJwq233ooVK1YgNDTUYeEmTpyIe++9FzExMcjOzsaLL76IMWPGICUlBTqd7rKvWbx4MV555RWHZSDxausbB/rmldc0zR/TO9JfdCwilyJJEl66swesDXas3mfEM98cQZ3Njt8MjhYdjcjpWnx8PiYmBhs2bEBZWRlOnz4NWZYRGxuLgADHz0I5ffr0pv/u2bMnBg4ciJiYGKxfvx7Tpk277GsWLFiA+fPnN902m82IiopyeDZqGxU19fghNQ9llnp4qFW4vRfXVyK6EkmS8OqUntCqJKzacxYLvjuGipp6PHZLF865RG6t1YMNAgICMGjQIEdmuabw8HDExMQgMzPzis/R6XRXPGpDypJfUYP/HslHTb0NPjoNpvSNQLAP/22JrkalkrBociJ0WjU+3HEGb2zMQG6pBa9MTuSyHuS2FPWTXVJSgtzc3BZdMUXKlFNSje8O5aGm3oYQXx2mD4pikSG6TpIk4YXbE/DSnT0gScCX+4x49LODqLZypmByT0LLTFVVFVJTU5GamgoAyM7ORmpqKoxGI6qqqvDMM89gz549yMnJwfbt2zFp0iQEBwfjrrvuEhmbnCyzsBL/PXIeDXYZnYK8cHf/SPjwiiWiFntoRGd88MAA6DQqbDtZjPv+uQeF5lrRsYgcTmiZOXjwIPr164d+/foBAObPn49+/frhpZdeglqtxrFjxzBlyhTExcVh1qxZiIuLw549e+Dr6ysyNjlR2vkKbEwrgF0G4kJ9cGfvCHhoFHUAkcil3NYzDGt+PxRB3h44ft6Myct24UBOqehYRA4l9M/dUaNGXXX67c2bN7dhGhLtkLEMOzNNABovvR7dPQQqDlokumH9ogOw9omb8LtP9yOruBr3f7gXz94aj9/f3AUqTm9AboB/8pJL2J9d2lRkBkQHYAyLDJFDRQd5Yd3cEZjSNwI2u4w3Nmbg0c8OotxSJzoa0Q1jmSHhUs6WYc+Zxll9h3UJwk3dgngZKZETeOs0WDq9L16/qxc8NCpsySjCHe/twv5snnYiZWOZIaGO5JZj1+nGIzLDugRhcOdAFhkiJ5IkCTOGRGPtE8PRKcgLeeU1mP7hHvz1xxOoredCraRMLDMkTFpeBbafalxlfVCnAAzuHCg4EVH7kRhhwH+fHIHpA6Mgy8DHu7Jx+7s7cchYJjoaUYuxzJAQGflmbMkoAgD0i/bHsC5BghMRtT++nlq8eU9vfPK7QQj10+GMqRr3fLAbr29Ih6WOc9KQcrDMUJvLNlXjpxOFAIDeHQ24uVswTy0RCTQ6PgQ/zRuJaf07wi4DH+44g/FLdmBrRqHoaETXhWWG2lShuRYb0/IhA0gI88Wo+A4sMkQuwOClxZL7+uLjWQPR0V+PvPIaPPTpQTz+eQryK2pExyO6KpYZajPmmnqsO3Ie9TYZUYF6jE0IZZEhcjFjE0KRNP8WPDayC9QqCZuOF2Dc35Pxr51n0GCzi45HdFksM9QmrPU2/HDkPCx1NgT5eOCOXuFQc7IuIpfk5aHBgokJWP/UCAyICUB1nQ2vrk/H5GW/IDW3XHQ8okuwzJDT2ewyfjyWj9LqOnjr1JjSJwI6jVp0LCK6hu5hfvjPY8PwxrReMOi1OJFvxl0rfsGL36fBXFsvOh5RE5YZcipZlrHtZBHOldVAq5YwpU9H+HpqRcciouukUkm4f3A0tvxpJKb16whZBj7fexZj/56M/x45f9UlaYjaCssMOVVanhnHz5shAbi9Zzg6+OpERyKiVgj20WHJ9L5Y/cgQdAn2RnGlFU9+dRizPjkAY4lFdDxq51hmyGnyK2qw/VTjXDLDuwahU7C34EREdKOGdwvGxnk3Y964WHioVdhxqhjj30nG8m2nUdfAAcIkBssMOUW1tQEbjhXALgNdO3hjQEyA6EhE5CA6jRrzxsVh07ybcVO3IFgb7Hh780lMXrYLx89XiI5H7ZBGdAByP3a7jI1pBaiyNiDAS4vxPXgJNilfenq66Ag3zGq1Qqdz7Kne+f20GBjkj09SzcgoqMTk93fh3h4+mJbgA42TrlgMDg5GdHS0U96blIllhhxuV5YJeeWNA37v7M0rl0jZzKWN64c9+OCDgpM4ggTAOQN2VV4GBE2YA6/44VhzvAqrfj6Mkg3voN5kdPjX0nt5ISM9nYWGmrDMkENlFVfhsLEcADChRxgCvT3EBiK6QTVVZgDAHY8tRHzvAYLTtF76/mRsXPWuU78PWQZyLQ1ILVMD4bGIfHg5egfY0MXHDkcdnC00ZuHLN5+FyWRimaEmLDPkMFXWBvyc3riWS/9of3QL8RGciMhxgiJiEBmbKDpGqxUaswA4//uIAtDnwmdBTokFqWUaVGt9MDYhhEdpyWk4AJgcQpZl/HS8ALX1doT46jC8a7DoSEQkiLdOg8l9InBzbDBUEpBZVIWv9ueiyFwrOhq5KZYZcohDxnLkltVAo5JwW2IYlyogauckSUL/6ADcMyASvp4aVNTU498HzyEtj1c7keOxzNANKzLXYneWCQAwMq4DAjhOhoguCDfoMWNwNLoEe8Mmy9iSUYTkU8Wwc+ZgciCWGboh9TY7Nh7/v/lkEiP8REciIhfjqVXjzt7hGNYlCACQmluOdUfOw9pgE5yM3AXLDN2QnZkmlFvq4aPTYGwC55MhosuTJAmDOwfi9p5h0KgknC2x4D8Hz6GihgtW0o1jmaFWM5ZacOzC+e8JPUKh1/JKBSK6uthQX9wzIBLeHmqUVNfh6wMcGEw3jmWGWqWuwY4tFy7D7h1pQFSgl+BERKQUoX6euH9QNDr46lBTb8O3h/KQV14jOhYpGMsMtcruLBPMtQ3w9dTgJl6GTUQt5OOpwd39O6Kjvx51Nju+P5yHsyXVomORQrHMUIvlldXgyLnG00vjEkLhoeGPERG1nE6jxpS+EYgJ8kKDXcZ/j+TjdFGV6FikQPwtRC1Sb7Mj6cLppZ4Rfojm6SUiugFatQqTekegW4gPbLKMDWn5yCgwi45FCsMyQy2y50wJKmoar14aEcvTS0R049QqCRMTw5AQ7gtZBn46UcgjNNQiLDN03QoqapsWkRzbneusEJHjqFQSxieEoke4H2QZ2JRWwDE0dN1YZui62O0ytmYUAQASwnzRKdhbcCIicjeSJGFsQkjTKacfj+bzKie6LiwzdF2OnCtHcZUVOo2Kp5eIyGlUUuP6bhcHBa9LPc95aOiaWGbomqpqG7DnTAkAYES3YHh5aAQnIiJ3plZJuKNXeNNl22tT81BmqRMdi1wYywxdU3JmMeptMsINnlx7iYjahFatwqQ+4Qjx1aG23o4fUs+jpp5rOdHlsczQVeWYqnG6qAqSBIyOD+HaS0TUZnQaNSb3iYCfpwYVNfX48eh52LjYNl0GywxdUYPNjm0nGwf99ovyRwdfneBERNTeeOs0mNwnAh5qFc6X1+JQCa+ipEuxzNAV7c8phbm2AT46DYZ0DhIdh4jaqSAfHW7vFQZJAowWNQzD7xcdiVwMywxdVrmlDofOlgMARsZ14JIFRCRUTJA3RseHAAD8b34QO428ZJv+D39D0WXtzDTBJsuICfRC1w6cU4aIxOvV0YBY38ZBwMsPlOP4+QrBichVsMzQJXJKqnHGVA2VBNwS14GDfonIZfTyt6Em6yDqbMAfvjiECku96EjkAlhmqBmbXcaOU8UAgD5R/gj09hCciIjo/0gSYPrxbwjxVsNYasG8rw/DbuclTu0dyww1c+RcOcos9dBr1RjSOVB0HCKiS9hrq/Dc8ADoNCpsO1mM97eeFh2JBGOZoSa1NmDfmVIAwE3dgriQJBG5rC4BWrw6tScAYOmWU03TSFD7xDJDTY6Xq1FnsyPEV4ce4Zzpl4hc270Do/DAkGjIMvD0V4eRW2oRHYkEEVpmduzYgUmTJiEiIgKSJOH7779v9rgsy1i0aBEiIiKg1+sxatQoHD9+XExYN+cR1g051Y1HYkbFc9AvESnDS5N6oG+UP8y1DXhqzWHU2+yiI5EAQstMdXU1+vTpg2XLll328bfeegtLlizBsmXLcODAAYSFhWH8+PGorKxs46TuTZZlBIx5BADQPcwX4Qa94ERERNdHp1Hj/d/0g6+nBoeN5Xgn6ZToSCSA0DIzceJEvPrqq5g2bdolj8myjKVLl2LhwoWYNm0aevbsiVWrVsFisWD16tUC0rqvfXlWeEb1hFqSMbwrZ/olImWJCvTCG9N6AwA+SM7CL6dNghNRW3PZMTPZ2dkoKCjAhAkTmu7T6XQYOXIkdu/efcXXWa1WmM3mZhtdWV2DHZ8fbdxHsb52+HpqBSciImq5O3qH4/5BUZBl4I9fp6Kkyio6ErUhly0zBQUFAIDQ0NBm94eGhjY9djmLFy+GwWBo2qKiopyaU+m+2HsW+VU22KrKEOdnEx2HiKjVXp6UiG4hPiiqtOKZ/xyBLHP+mfbCZcvMRb8eiCrL8lUHpy5YsAAVFRVNW25urrMjKlaFpR7vbc0EAJTv+gJal/9pICK6Mr1H4/gZjwvzz6z8JUd0JGojLvvrKywsDAAuOQpTVFR0ydGa/6XT6eDn59dso8tbti0T5ZZ6RPlpUHU0SXQcIqIblhDuh7/ckQAAeHNjBk4W8IKR9sBly0znzp0RFhaGpKT/+yVbV1eH5ORkDB8+XGAy92AssWDV7rMAgFl9fAGZlzMSkXuYOTQGo+M7oM5mxx+/TkVdAz/f3J3QMlNVVYXU1FSkpqYCaBz0m5qaCqPRCEmSMG/ePLz++utYu3Yt0tLSMHv2bHh5eWHGjBkiY7uFNzdloM5mx82xwegf7ik6DhGRw0iShDfv7o0ALy1O5Jvx3pZM0ZHIyYSWmYMHD6Jfv37o168fAGD+/Pno168fXnrpJQDAc889h3nz5uGJJ57AwIEDkZeXh59++gm+vr4iYyteytlSrD+WD5UELLxwOJaIyJ2E+Hnitbt6AQBWbD+NlLNlghORMwktM6NGjYIsy5dsn376KYDGdr1o0SLk5+ejtrYWycnJ6Nmzp8jIiifLMl5dnw4AuHdAFLqHcUwREbmn23uF465+HWGXgT/9OxWWugbRkchJXHbMDDnH+mP5OGwsh5eHGn+aECc6DhGRUy2anIhwgydySix4fUO66DjkJCwz7Yi1wYY3N2UAAB67pStC/DhWhojcm0Gvxdv39AEAfLHXiB2nigUnImdgmWlHVu3OQW5pDUL9dHj0ls6i4xARtYkRscGYPbwTAOD5b4+isrZebCByOJaZdqK0ug7vbz0NAPjThHh4eWgEJyIiajvP3RaP6EAvnK+oxRsbM0THIQdjmWkn3tuSicraBiSE++Hu/pGi4xARtSkvDw3euLvx6qYv9xmxm4tRuhWWmXbgTHEVvtjbOEHeX+5IgFp15eUgiIjc1fCuwXhgSDQA4M/fHeXVTW6EZaYdeGNjBhrsMsZ0D8FN3YJFxyEiEmbB7Qno6K9HbmkN3tp0UnQcchCWGTe390wJfjpRCLVKwgu3dxcdh4hIKB+dBq9PazzdtGpPDg7klApORI7AMuPG7HYZr12YIO/+QVHoFsKZk4mIRsZ1wH0DIyHLwJ+/OYraepvoSHSDWGbc2A9H8nAsrwI+Og3+OJ4T5BERXbTwjh4I9dPhjKkaS5JOiY5DN4hlxk3V1tvw9oXzwX8Y1RXBPjrBiYiIXIdBr8XrF9Zu+tfOMzhs5NpNSsYy46Y+3pWN8xW16Oivx8MjOEEeEdGvjU0IbVq76blvjsLawNNNSsUy44ZMVVZ8sD0LAPDsrfHw1KoFJyIick0v3dkDwT46ZBZVYfmFiUVJeVhm3NA7SadQZW1A70gDJveJEB2HiMhlBXh74K9TEgEAK7ZnIT3fLDgRtQbntHczpwor8dV+IwDghdsToOIEeUTkhtLTHbcCdiiAoR09sTevFnM/24s3xga1yeSiwcHBiI6OdvrXaQ9YZtzMq+vTYZeBWxNDMbRLkOg4REQOZS5tXPX6wQcfdOj7qr0DEP7IB8gq88G4Oa/BvH+tQ9//cvReXshIT2ehcQCWGTey7WQRdpwqhlYtYcHEBNFxiIgcrqaq8TTQHY8tRHzvAQ5975wqFVJKgaAxD+H+GTPho3Xo2zdTaMzCl28+C5PJxDLjACwzbqLeZm+aIG/28E7oFOwtOBERkfMERcQgMjbRoe/ZUZZRdDgPuWU1SKsx4O6EjpAknqpXAg4AdhNf7TfidFEVAr09MHdMrOg4RESKI0kSxiaEQqOSkFdeg7TzHAysFCwzbqDCUo93Lsxg+cfxcTDonXhslIjIjRn0Wgzv2jjecFemCZW19YIT0fVgmXED72/NRJmlHnGhPvjNoCjRcYiIFK1PlD/C/DxRZ7Nj28liyLIsOhJdA8uMwmWbqrFqTw6AxrVGNGr+kxIR3QiVJGFcQghUUuNn7KnCKtGR6Br4m0/hXt+QjnqbjFHxHTAyroPoOEREbiHIR4fBnQIBAMmnilFTx6UOXBnLjILtzjIh6UQh1CoJf7mDl2ITETnSwE6BCPL2QE29DTsyi0XHoatgmVEom13GX39svBT7gSHR6BbiKzgREZF7UaskjEsIhQQgo6AS2aZq0ZHoClhmFOqblFyk55vh66nBvHFxouMQEbmlMIMn+kX7AwC2ZhRxZW0XxTKjQObaery9+SQA4OmxsQj09hCciIjIfQ3tEgSDXosqawN2ny4RHYcug2VGgZYmZcJUVYeuHbzx22GdRMchInJrWrUKY7uHAACO5lUgr6xGcCL6NZYZhTlZUNl0KfaiyYnw0PCfkIjI2aICvZAY4QcA+Dm9EA02u+BE9L/4m1BBZFnGy+vSYLPLuC0xDDfH8lJsIqK2cnO3YHh7qFFeU4992aWi49D/YJlRkB+P5mPvmVLoNCr85U5eik1E1JZ0WjVGXzjdlGIsQ5G5VnAiuohlRiGqrQ1Nq2LPGd0NkQFeghMREbU/XTv4IDbEB7IM/JxeBJudSx24ApYZhVi+7TQKzLWICtTj97d0ER2HiKjdGhnXAZ4aFYqrrEgxlomOQ2CZUYQzxVX4aOcZAMBLdybCU6sWnIiIqP3y1mlwy4XlY/afKYWpyio4EbHMuDhZlrFwbVrT+kvjEkJERyIiave6h/mic7A3bLKMpBOFPN0kGMuMi/vuUB72nCmBp1aFv07pCUmSREciImr3JEnC2O4h0GlUKKq04mAOr24SiWXGhZVW1+HV9ScAAE+PjUNUIAf9EhG5Cm+dBqPjG4+W788pRVElr24ShWXGhS3ekI4ySz3iQ33xyM2dRcchIqJfiQv1QbcOPrDLwE8nCtFg52R6IrDMuKg9WSX4T8o5AMDr03pBq+Y/FRGRq5EkCaO7d4Beq0ZJVR32czI9Ifgb0gVZG2xY+P0xAMADQ6IxICZAcCIiIroSLw8NxlyYTO9gThnyK7h2U1tjmXFBH2zPwpniagT76PDcbd1FxyEiomvoFuKD+DBfyAA2Hy9EXQNPN7UllhkXk55vxvJtpwEAL0/qAYNeKzgRERFdj9FxHeDrqUFFTT2STxWLjtOusMy4kHqbHc9+cwT1Nhnje4Tizt7hoiMREdF10mnVmNAjFABwIt+M00VVghO1HywzLuQf27OQlmeGQa/Fa3dxThkiIqWJDPBqGue4Jb0QVdYGwYnaB5cuM4sWLYIkSc22sLAw0bGcIj3fjPe2ZgIAXpmciBBfT8GJiIioNYZ1CUKIrw61DXYknSiELHN2YGdz6TIDAImJicjPz2/ajh07JjqSw/369NKUvhGiIxERUSupVRJuTQyDRiXBWGrB4dxy0ZHcnkZ0gGvRaDRuezTmIp5eIiJyL4HeHrg5NhjbThZj9+kSdPTXI9SPR9ydxeWPzGRmZiIiIgKdO3fG/fffjzNnzlz1+VarFWazudnmyk6c5+klIiJ31KujAV07NC5GueFYPqz1NtGR3JZLl5khQ4bgs88+w+bNm/HRRx+hoKAAw4cPR0lJyRVfs3jxYhgMhqYtKiqqDRO3TE2dDU+tOYx6m4wJPL1ERORWJEnCuIRQ+HlqYK5tQFI6x884i0uXmYkTJ+Luu+9Gr169MG7cOKxfvx4AsGrVqiu+ZsGCBaioqGjacnNz2ypui726/gROF1UhxFeHN+7uzdNLRERuxlOrxsSe4VBJQFZxNY6cqxAdyS25/JiZ/+Xt7Y1evXohMzPzis/R6XTQ6XRtmKp1Nh8vwJf7jACAJff1RaC3h+BERETkDGEGT4zoFowdmSbszCxGuIHDCRzNpY/M/JrVakV6ejrCw5U9mVxBRS3+/O1RAMBjt3TBiNhgwYmIiMiZ+kb5o2sHb9hlYMOxfNRxtQOHcuky88wzzyA5ORnZ2dnYt28f7rnnHpjNZsyaNUt0tFaz2WX88etUlFvq0bOjH/40IV50JCIicrJfj585YNIAkkv/ClYUl96T586dw29+8xvEx8dj2rRp8PDwwN69exETEyM6Wqv9c0cW9pwpgZeHGu/d3w8eGpf+JyAiIgfx1Kpxe69wqFUSCmpV8B/xgOhIbsOlx8ysWbNGdASH2pNVgr//dAoAsGhyIrp08BGciIiI2lKonyfGdQ/B5hOFMAyfjt25NejfX3Qq5eNhgTaSX1GDuasPwWaXMa1fR9w7IFJ0JCIiEqB7uB9ifRvnnHl/fwXS8117PjQlYJlpA9YGG5748hBKquuQEO6H1+7qxcuwiYjasZ7+NtTkHIbVJuP3nx9EuaVOdCRFY5lpA3/98QQOG8vh56nBPx8cAL2HWnQkIiISSCUBph/eQqi3GrmlNZiz+hDqbbzEqbVcesyMEhiNRphMpis+vi3Hgi/2V0AC8OQgX5jOZsB0tu3yXY/09HTREYiI2h17bSWevykAC7eX4ZfTJXj+22P4272cQLU1WGZugNFoRPeEBNRYLJd93CO0K0IfeAsqrQ5lu77E79/8qo0TtkxVVZXoCERE7UqMvxbLH+iPR1YdxLeHzqFjgB7zx8eJjqU4LDM3wGQyocZiwQN/fhuh0V2bPWZpALYVaFFrlxDmace039wLaca9gpJeXfr+ZGxc9S5qa2tFRyEiandGx4fg1ak9seC7Y3hvSyYi/fW4b5DrrivoilhmHCA0uisiYxObblsbbPhPyjnU2usQ5O2BqQMjodO47jiZQmOW6AhERO3abwZHI6+sBsu2ncaCtccQavDEyLgOomMpBgcAO5jNLmPjsQKUVNXBy0ONyX0jXLrIEBGRa/jThDhM69cRNruMJ75IwTEuSnndWGYcSJZlbD9VhLOlFmhUEib3iYCfp1Z0LCIiUgBJkvDG3b1xU7cgVNfZMHPlPmQUcA6a68Ey40CHjOVIy2v8wbutZxhC/bgyKhERXT8PjQr/eHAA+kb5o9xSjwc+2ofTRZWiY7k8lhkHScurwK7TjZdo3xIbjK5cqoCIiFrB11OLVQ8NRs+OfiiprsOMj/Yhx1QtOpZLY5lxgNxqFbZkFAEA+kf7o2+Uv9hARESkaAa9Fp8/NATdw3xRVGnFjI/2Irf08tOAEMvMDdN3G4IDJY0DfHt1NGBEt2BOeERERDcswNsDnz88BF07eON8RS1+89FeHqG5ApaZG5BaYEWHKc9DhoTuYb4YHd+BRYaIiBymg68Oqx8dis7B3jhXVoN7/rEHJ85zUPCvscy00oGcUrz5SxkkjRYRejvGJ4SyyBARkcOF+nni348NQ49wP5iqrJj+4R4cyCkVHculsMy00n+PnIfVJqPmzEEMCW6ASsUiQ0REztHBV4evfj8UgzoFoLK2ATM/3odtF8ZqEstMqy2alIiH+/qheO1isMcQEZGzGfRafPbQEIzpHoLaejse/ewg/n0wV3Qsl8Ay00oqlYQ74rwhN1hFRyEionZC76HGP2cOwNS+EWiwy3jum6P4648n0GCzi44mFMsMERGRgmjVKiy5ry+eGhsLAPh4VzYeWnUQFTX1gpOJwzJDRESkMCqVhPnj47B8Rn94alXYcaoYdy3/BVnFVaKjCcEyQ0REpFB39A7HN48PR4TBE2dM1Zi67Bf8kJonOlabY5khIiJSsJ4dDfhh7ojGK52sDXh6TSqe+c8RVFsbREdrMywzRERECtfBV4evHh2Kp8bGQiUB36Scw53v70JaXoXoaG2CZYaIiMgNaNQqzB8fh68eHYpwgyeyTdW4a8UvWLY1E3UN7n21E8sMERGRGxnSJQgbn74ZtyWGod4m428/ncKk93ch5WyZ6GhOoxEdgIiIqL1KT0932ns/2gPo7uOPlalmnCysxD0f7MatXb3wQC9feHs47lhGcHAwoqOjHfZ+rcEyQ0RE1MbMpcUAgAcffNDpX0vl6YuAMQ/Bp9d4bMqy4Mcj51Cx8wtUHfsZkG/89JPeywsZ6elCCw3LDBERURurqWpc+fqOxxYivveANvmaRbX1OFSqQbVPIIImPoXOk+eil78NoZ4yWrtOcqExC1+++SxMJhPLDBERUXsUFBGDyNjENvlakQB62+04eq4C+7NLYa4HfilWISpAj+FdgxFm8GyTHM7AMkNERNROaFQq9I8OQI9wPxzIKcWR3ArkltXg64O5iArQY1CnQEQG6CG19lCNICwzRERE7YynVo2bYzugT6Q/9maX4GRBJXLLapBblocwP08M6hSATsHeUCmk1LDMEBERtVN+ei0m9AjD0M5BSDGW4fh5MwrMtfjv0Xz4eWrQK9KAxHAD9B5q0VGvimWGiIionfPTazE6PgSDOwXicG450vIqYK5twC+nS7D3TCniQnyQ2NGACIOnS56CYpkhIiIiAIC3ToMR3YIxpHMgThVW4ui5ChRVWpFeUIn0gkr4eWrQPdwP3cN8EeDlITpuE5YZIiIiakarViExwoAe4X4oNFtxNK8cp4uqYK5twP7sUuzPLkWYnyc6qFRQ+3YQHZdlhoiIiC5PkiSEGTwRZgjD6Hg7soqrkFFQCWOJBQXmWhRAg8gnPsFHhyrwQX9xOVlmiIiI6Jq0ahW6h/mhe5gfqq0NOF1UhbSzhSiuBaL9xNYJLjRJRERELeKt06BPlD9Ghjbg3PLf4uYYvdA8LDNERETUavbqcnhpxdYJlhkiIiJSNJYZIiIiUjSWGSIiIlI0lhkiIiJSNJYZIiIiUjRFlJkVK1agc+fO8PT0xIABA7Bz507RkYiIiMhFuHyZ+frrrzFv3jwsXLgQhw8fxs0334yJEyfCaDSKjkZEREQuwOXLzJIlS/Dwww/jkUceQUJCApYuXYqoqCh88MEHoqMRERGRC3Dp5Qzq6uqQkpKC559/vtn9EyZMwO7duy/7GqvVCqvV2nS7oqICAGA2mx2er6qqCgBwLvM4rDUWh79/Wyk0ZgEACnJOIcvbS3Ca1uP34Vr4fbgWfh+uxV2+j+Jz2QAafx86+vfsxfeTZfnaT5ZdWF5engxA/uWXX5rd/9prr8lxcXGXfc3LL78sA+DGjRs3bty4ucGWm5t7zb7g0kdmLpIkqdltWZYvue+iBQsWYP78+U237XY7SktLERQUdMXX0JWZzWZERUUhNzcXfn5+ouO0O9z/4nDfi8N9L44r7XtZllFZWYmIiIhrPtely0xwcDDUajUKCgqa3V9UVITQ0NDLvkan00Gn0zW7z9/f31kR2w0/Pz/hP9jtGfe/ONz34nDfi+Mq+95gMFzX81x6ALCHhwcGDBiApKSkZvcnJSVh+PDhglIRERGRK3HpIzMAMH/+fMycORMDBw7EsGHD8OGHH8JoNOLxxx8XHY2IiIhcgMuXmenTp6OkpAT/7//9P+Tn56Nnz57YsGEDYmJiREdrF3Q6HV5++eVLTt1R2+D+F4f7Xhzue3GUuu8lWb6ea56IiIiIXJNLj5khIiIiuhaWGSIiIlI0lhkiIiJSNJYZIiIiUjSWGcKKFSvQuXNneHp6YsCAAdi5c+cVn7t9+3ZIknTJlpGR0YaJ3UdL9j3QuPbYwoULERMTA51Oh65du2LlypVtlNb9tGT/z549+7I/+4mJiW2Y2H209Gf/yy+/RJ8+feDl5YXw8HD87ne/Q0lJSRuldS8t3ffLly9HQkIC9Ho94uPj8dlnn7VR0ha44QWUSNHWrFkja7Va+aOPPpJPnDghP/3007K3t7d89uzZyz5/27ZtMgD55MmTcn5+ftPW0NDQxsmVr6X7XpZlefLkyfKQIUPkpKQkOTs7W963b98la5fR9Wnp/i8vL2/2M5+bmysHBgbKL7/8ctsGdwMt3fc7d+6UVSqV/O6778pnzpyRd+7cKScmJspTp05t4+TK19J9v2LFCtnX11des2aNnJWVJX/11Veyj4+PvG7dujZOfnUsM+3c4MGD5ccff7zZfd27d5eff/75yz7/YpkpKytrg3TuraX7fuPGjbLBYJBLSkraIp7ba+n+/7W1a9fKkiTJOTk5zojn1lq6799++225S5cuze5777335MjISKdldFct3ffDhg2Tn3nmmWb3Pf300/JNN93ktIytwdNM7VhdXR1SUlIwYcKEZvdPmDABu3fvvupr+/Xrh/DwcIwdOxbbtm1zZky31Jp9v27dOgwcOBBvvfUWOnbsiLi4ODzzzDOoqalpi8hu5UZ+9i/6+OOPMW7cOE7g2UKt2ffDhw/HuXPnsGHDBsiyjMLCQnzzzTe444472iKy22jNvrdarfD09Gx2n16vx/79+1FfX++0rC3FMtOOmUwm2Gy2SxbtDA0NvWRxz4vCw8Px4Ycf4ttvv8V3332H+Ph4jB07Fjt27GiLyG6jNfv+zJkz2LVrF9LS0rB27VosXboU33zzDebMmdMWkd1Ka/b//8rPz8fGjRvxyCOPOCui22rNvh8+fDi+/PJLTJ8+HR4eHggLC4O/vz/ef//9tojsNlqz72+99Vb861//QkpKCmRZxsGDB7Fy5UrU19fDZDK1Rezr4vLLGZDzSZLU7LYsy5fcd1F8fDzi4+Obbg8bNgy5ubn429/+hltuucWpOd1RS/a93W6HJEn48ssvm1aSXbJkCe655x4sX74cer3e6XndTUv2///69NNP4e/vj6lTpzopmftryb4/ceIEnnrqKbz00ku49dZbkZ+fj2effRaPP/44Pv7447aI61Zasu9ffPFFFBQUYOjQoZBlGaGhoZg9ezbeeustqNXqtoh7XXhkph0LDg6GWq2+pJEXFRVd0tyvZujQocjMzHR0PLfWmn0fHh6Ojh07NhUZAEhISIAsyzh37pxT87qbG/nZl2UZK1euxMyZM+Hh4eHMmG6pNft+8eLFuOmmm/Dss8+id+/euPXWW7FixQqsXLkS+fn5bRHbLbRm3+v1eqxcuRIWiwU5OTkwGo3o1KkTfH19ERwc3BaxrwvLTDvm4eGBAQMGICkpqdn9SUlJGD58+HW/z+HDhxEeHu7oeG6tNfv+pptuwvnz51FVVdV036lTp6BSqRAZGenUvO7mRn72k5OTcfr0aTz88MPOjOi2WrPvLRYLVKrmv64uHhWQubzgdbuRn3utVovIyEio1WqsWbMGd9555yX/JkIJGnhMLuLiZXoff/yxfOLECXnevHmyt7d30xUazz//vDxz5sym57/zzjvy2rVr5VOnTslpaWny888/LwOQv/32W1HfgmK1dN9XVlbKkZGR8j333CMfP35cTk5OlmNjY+VHHnlE1LegaC3d/xc9+OCD8pAhQ9o6rltp6b7/5JNPZI1GI69YsULOysqSd+3aJQ8cOFAePHiwqG9BsVq670+ePCl//vnn8qlTp+R9+/bJ06dPlwMDA+Xs7GxB38HlscyQvHz5cjkmJkb28PCQ+/fvLycnJzc9NmvWLHnkyJFNt9988025a9eusqenpxwQECCPGDFCXr9+vYDU7qEl+16WZTk9PV0eN26crNfr5cjISHn+/PmyxWJp49Tuo6X7v7y8XNbr9fKHH37YxkndT0v3/XvvvSf36NFD1uv1cnh4uPzAAw/I586da+PU7qEl+/7EiRNy3759Zb1eL/v5+clTpkyRMzIyBKS+OkmWeYyOiIiIlMuFTngRERERtRzLDBERESkaywwREREpGssMERERKRrLDBERESkaywwREREpGssMERERKRrLDBERESkaywwREREpGssMERERKRrLDBERESkaywwREREp2v8Hi3lyAQBvWkgAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(simsvalues, kde=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "f993f290-677a-4a61-93b7-0977d7534d7b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "573f3968-56dc-4d93-9b92-2924019993c0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    126.000000\n",
       "mean       0.655297\n",
       "std        0.099786\n",
       "min        0.439509\n",
       "25%        0.583280\n",
       "50%        0.661202\n",
       "75%        0.720619\n",
       "max        0.912604\n",
       "dtype: float64"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(simsvalues).describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8715caf-e85a-4690-beb2-0d732a8b244d",
   "metadata": {
    "tags": []
   },
   "source": [
    "## 将question和answer进行cross 对比，生成126*126个结果，再查看这个分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "e529a082-e4d0-4f0e-85bd-f38160a09cfb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cross_simsvalues = util.cos_sim(emb_answer,emb_question).flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "23cfc0f1-4468-48bb-8349-1011605d5870",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cross_sims_s = pd.Series(cross_simsvalues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "cdd17e5a-01cc-481b-9f36-e56944f2cd86",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "N = len(input_question)\n",
    "pos_indices = [ i*N+i for i in range(N)] ##正例的index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "92f376bf-da76-4f80-bf93-ca0165f2dbbf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    126.000000\n",
       "mean       0.655297\n",
       "std        0.099786\n",
       "min        0.439509\n",
       "25%        0.583280\n",
       "50%        0.661202\n",
       "75%        0.720619\n",
       "max        0.912604\n",
       "dtype: float64"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 正样本的得分\n",
    "pos_cross_sims_s = cross_sims_s[pos_indices]\n",
    "pos_cross_sims_s.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "a1b58af3-47be-423a-8999-da07609eaa9f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    15750.000000\n",
       "mean         0.099584\n",
       "std          0.123988\n",
       "min         -0.246504\n",
       "25%          0.015882\n",
       "50%          0.086160\n",
       "75%          0.162934\n",
       "max          0.813635\n",
       "dtype: float64"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#负样本得分\n",
    "neg_cross_sims_s = cross_sims_s.drop(pos_indices)\n",
    "neg_cross_sims_s.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "98bfbea3-1274-4ea0-91d9-69af1784600c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: ylabel='Count'>"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjMAAAGdCAYAAADnrPLBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABik0lEQVR4nO3dd3xV9f3H8dfNJJtAEhIghAAJe4MMAQERR7UKWrUu7K9aF1VK624V+6vS2upPW8W2bgW34gZE2VP2HgECYSRAQvYe5/fHIQFkJtyb7x3v5+NxH/fkjnPfHhPyyXc6LMuyEBEREfFQfqYDiIiIiJwPFTMiIiLi0VTMiIiIiEdTMSMiIiIeTcWMiIiIeDQVMyIiIuLRVMyIiIiIR1MxIyIiIh4twHQAV6upqeHAgQNERETgcDhMxxEREZFzYFkWhYWFtGzZEj+/M7e9eH0xc+DAARITE03HEBERkQbYu3cvrVu3PuNrvL6YiYiIAOyLERkZaTiNiIiInIuCggISExPrfo+fidcXM7VdS5GRkSpmREREPMy5DBExOgD4lVdeoUePHnWFxqBBg5gxY0bd87fffjsOh+OE28CBAw0mFhEREXdjtGWmdevW/PWvf6VDhw4AvP3221x99dWsWbOGrl27AnDZZZfx5ptv1r0nKCjISFYRERFxT0aLmauuuuqEr59++mleeeUVli1bVlfMBAcHEx8fbyKeiIiIeAC3WWemurqaDz74gOLiYgYNGlT3+Lx584iLiyM1NZU777yTQ4cOnfE85eXlFBQUnHATERER72W8mNmwYQPh4eEEBwdz9913M336dLp06QLA5ZdfzrRp05gzZw7PPfccK1asYOTIkZSXl5/2fJMnTyYqKqrupmnZIiIi3s1hWZZlMkBFRQUZGRnk5eXx6aef8tprrzF//vy6guZ4mZmZJCUl8cEHHzB27NhTnq+8vPyEYqd2ald+fr5mM4mIiHiIgoICoqKizun3t/Gp2UFBQXUDgPv168eKFSt48cUX+c9//nPSaxMSEkhKSiItLe205wsODiY4ONhleUVERMS9GO9m+inLsk7bjZSTk8PevXtJSEho5FQiIiLiroy2zDz22GNcfvnlJCYmUlhYyAcffMC8efOYOXMmRUVFTJo0iWuvvZaEhAR2797NY489RkxMDGPGjDEZW0RERNyI0WLm4MGD3HrrrWRmZhIVFUWPHj2YOXMml1xyCaWlpWzYsIF33nmHvLw8EhISGDFiBB9++OE5LW0sIiIivsH4AGBXq88AIhEREXEP9fn97XZjZkRERETqQ8WMiIiIeDTjU7PF+2RkZJCdne2Sc8fExNCmTRuXnFtERDyTihlxqoyMDDp36kRJaalLzh8aEsKWrVtV0IiISB0VM+JU2dnZlJSWMnXMGDrHxjr13FsOH+aW6dPJzs5WMSMiInVUzIhLdI6NpY8WNxQRkUagAcAiIiLi0VTMiIiIiEdTMSMiIiIeTcWMiIiIeDQVMyIiIuLRVMyIiIiIR1MxIyIiIh5NxYyIiIh4NBUzIiIi4tFUzIiIiIhHUzEjIiIiHk3FjIiIiHg0FTMiIiLi0VTMiIiIiEdTMSMiIiIeTcWMiIiIeDQVMyIiIuLRAkwHEBERqZWRkUF2drbTzxsTE0ObNm2cfl5xDypmRETELWRkZNCpcydKS0qdfu6Q0BC2btmqgsZLqZgRERG3kJ2dTWlJKWMeG0NsUqzTznt4z2GmPzOd7OxsFTNeSsWMiIi4ldikWBJSE0zHEA+iAcAiIiLi0VTMiIiIiEdTMSMiIiIeTcWMiIiIeDQVMyIiIuLRVMyIiIiIR1MxIyIiIh5NxYyIiIh4NBUzIiIi4tFUzIiIiIhHUzEjIiIiHk3FjIiIiHg0FTMiIiLi0VTMiIiIiEczWsy88sor9OjRg8jISCIjIxk0aBAzZsyoe96yLCZNmkTLli0JCQlh+PDhbNq0yWBiERERcTdGi5nWrVvz17/+lZUrV7Jy5UpGjhzJ1VdfXVewPPvsszz//PO89NJLrFixgvj4eC655BIKCwtNxhYRERE3YrSYueqqq7jiiitITU0lNTWVp59+mvDwcJYtW4ZlWbzwwgs8/vjjjB07lm7duvH2229TUlLCe++9ZzK2iIiIuBG3GTNTXV3NBx98QHFxMYMGDSI9PZ2srCxGjx5d95rg4GAuuugilixZctrzlJeXU1BQcMJNREREvJfxYmbDhg2Eh4cTHBzM3XffzfTp0+nSpQtZWVkAtGjR4oTXt2jRou65U5k8eTJRUVF1t8TERJfmFxEREbOMFzMdO3Zk7dq1LFu2jHvuuYdx48axefPmuucdDscJr7cs66THjvfoo4+Sn59fd9u7d6/LsouIiIh5AaYDBAUF0aFDBwD69evHihUrePHFF3n44YcByMrKIiEhoe71hw4dOqm15njBwcEEBwe7NrSIiIi4DeMtMz9lWRbl5eUkJycTHx/P7Nmz656rqKhg/vz5DB482GBCERERcSdGW2Yee+wxLr/8chITEyksLOSDDz5g3rx5zJw5E4fDwYQJE3jmmWdISUkhJSWFZ555htDQUG666SaTsUVERMSNGC1mDh48yK233kpmZiZRUVH06NGDmTNncskllwDw0EMPUVpayr333ktubi4DBgzgu+++IyIiwmRsERERcSNGi5nXX3/9jM87HA4mTZrEpEmTGieQiIiIeBy3GzMjIiIiUh8qZkRERMSjqZgRERERj6ZiRkRERDyaihkRERHxaCpmRERExKOpmBERERGPpmJGREREPJqKGREREfFoKmZERETEo6mYEREREY+mYkZEREQ8mooZERER8WgqZkRERMSjqZgRERERj6ZiRkRERDyaihkRERHxaCpmRERExKOpmBERERGPpmJGREREPJqKGREREfFoKmZERETEo6mYEREREY+mYkZEREQ8mooZERER8WgqZkRERMSjqZgRERERj6ZiRkRERDyaihkRERHxaCpmRERExKOpmBERERGPpmJGREREPJqKGREREfFoKmZERETEo6mYEREREY+mYkZEREQ8mooZERER8WgqZkRERMSjqZgRERERj6ZiRkRERDya0WJm8uTJ9O/fn4iICOLi4rjmmmvYtm3bCa+5/fbbcTgcJ9wGDhxoKLGIiIi4G6PFzPz587nvvvtYtmwZs2fPpqqqitGjR1NcXHzC6y677DIyMzPrbt9++62hxCIiIuJuAkx++MyZM0/4+s033yQuLo5Vq1YxbNiwuseDg4OJj49v7HgiIuJGqmuq8XP44XA4TEcRN2O0mPmp/Px8AJo1a3bC4/PmzSMuLo6mTZty0UUX8fTTTxMXF3fKc5SXl1NeXl73dUFBgesCi4iIS5VVlbFgzwI2HNpAUUURYYFh9GjRg2FJw2gS0MR0PHETbjMA2LIsJk6cyJAhQ+jWrVvd45dffjnTpk1jzpw5PPfcc6xYsYKRI0eeULAcb/LkyURFRdXdEhMTG+s/QVyhoAAOHIDcXLAs02lEpBEdKj7EKytfYem+pRRVFAFQXFnM0n1L+e+q/5Jbmms4obgLt2mZGT9+POvXr2fRokUnPH7DDTfUHXfr1o1+/fqRlJTEN998w9ixY086z6OPPsrEiRPrvi4oKFBB42ksCzZsgIULITu77uEuUVH8DqCy0lg0EWkc2SXZvLn2TcqqyohuEs3o9qNJjExkX8E+ZuyYQW5ZLu+sf4ff9PkNIYEhpuOKYW5RzPz2t7/lyy+/ZMGCBbRu3fqMr01ISCApKYm0tLRTPh8cHExwcLArYkpjKC+Hjz6CXbvsrx0OCAuD0lKa5OfzPFD8q1/BF19ASorRqCLiGuU15Xy84WPKqspoFdGKm7vfXFewdIzpSMuIlry59k1yy3L5fOvn3NjtRo2j8XFGu5ksy2L8+PF89tlnzJkzh+Tk5LO+Jycnh71795KQkNAICaVRlZTAO+/YhUxAAIwcCQ89BL//PTz0EBlDhnAECNuyBYYMgY0bTScWERdYmL+Q3LJcmjZpyi+7/fKklpeI4Aiu73o9/g5/th/ZzqbDmwwlFXdhtJi57777mDp1Ku+99x4RERFkZWWRlZVFaWkpAEVFRfzhD39g6dKl7N69m3nz5nHVVVcRExPDmDFjTEYXZ6upgU8+scfHhIbC//wPDB0KTY4O8AsKIrtLF7oDJR07wqFDMGIE7NxpNLaIOFkq7CrbhZ/Dj+s6X0dYUNgpXxYfHs+QNkMA+G7nd1RWq/vZlxktZl555RXy8/MZPnw4CQkJdbcPP/wQAH9/fzZs2MDVV19Namoq48aNIzU1laVLlxIREWEyujjbnDmQng6BgTBuHJym5e0AkPbKK9Cnjz2e5pproKioUaOKiGuUV5fDZfbxwNYDaRXZ6oyvH9JmCJHBkRRWFLIqc1UjJBR3ZXTMjHWW2SkhISHMmjWrkdKIMXv3wuLF9vHVV8Nppt3Xqo6Kgq++gr597a6mu++GqVMbIaiIuNI7O9+BZhDqF8qwNsPO+voAvwCGJQ3j6+1fs3jvYvq17EeAn1sMBZVG5jZTs8VHVVfD11/bx716Qdeu5/a+li3h00/B3x+mTYPPPnNZRBFxvczCTN5MexOAgZEDCQ44t4kcvVr0IiIogqKKIjYd0tgZX6ViRsxavtwe/xIaCpdcUr/3Dh4MDz9sH9999wnTuEXEszy7+FnKa8phL7Rv0v6c3+fv588FrS4AYPn+5a6KJ25OxYyYU14OtesKXXyxXdDU1xNPQLducPgw/PGPzs0nIo0iszCTf6/6t/3FPOo9zbpPQh/8HH5kFmVyqPiQ8wOK21MxI+YsXQqlpdC8ud3F1BDBwfDyy/bxf/8Lq1c7LZ6INI5nFz9LWVUZPaJ7QAMmKIYGhpLaPBWAtVlrnRtOPIKKGTGjrAyWLbOPhw8Hv/P4Vhw2DH75S3vl4N/9TtseiHiQw8WH61plfpP6mwafp2eLngBsOLSBGqvGKdnEc6iYETNWr7a7mWJjz33Q75n87W92K82CBfCT3dhFxH1NWTGFsqoy+rXsx8DYgQ0+T0qzFEICQiiqKGJX7i4nJhRPoGJGGl91tT3wF2DgQHvLgvOVmAjjx9vHjz1mL8InIm6ttLKUl1fY3cS/H/T789qSwN/Pn+4tugOwLmudU/KJ51AxI41vyxZ7N+ywMOjRw3nnffRRiIyEtWth+nTnnVdEXGLq+qkcLjlMm6g2XNfluvM+X484+9+T7Ue2U1VTdd7nE8+hYkYa34oV9n3//vYeTM7SvDncf799PHmyxs6IuLEaq4bnlz0PwIQBE5yy2F3LiJZEBEVQUV1Bem76eZ9PPIeKGWlchw9DRobdtdSnj/PPf//9EBICq1bB9987//wi4hRz0uewNXsrEUER/LrPr51yTofDUTeraVvONqecUzyDihlpXLVTp1NTwRX7a8XGwp132seTJzv//CLiFLVjZcb1HEdkcKTTztspphNgFzNn2zJHvIeKGWk8VVWw7ujAPFe0ytT6/e/t7qu5c48NNBYRt5GRn8GX274E4J7+9zj13G2btiXIP4iiiiIOFB5w6rnFfamYkcaTlmYvkhcRAR06uO5z2rSBW26xj9U6I+J2/rvqv9RYNYxoO4IusV2ceu4AvwA6RNv/vmzN2erUc4v7UjEjjWfDBvu+W7fzWyTvXDz8sD0u54svYPNm136WiJyz8qpyXl39KgD39b/PJZ/RMaYjAGk5aS45v7gfFTPSOMrKYPt2+9iZ07FPp1MnuOYa+/hf/3L954nIOfl0y6ccKj5Ey4iW/Lzjz13yGe2j7Y0qDxYfpKiiyCWfIe5FxYw0js2b7cXy4uKgRYvG+cwHHrDv33kHcnMb5zNF5IymrJgCwF197yLQP9AlnxEWFEZ8eDyAVgP2ESpmpHFs2mTfd+/unBV/z8WwYXYrUEkJvPFG43ymiJzWuqx1LN67mAC/AO7sc6dLP6tddDtAxYyvUDEjrldaCrt328ddnDvY74wcjmOL6L30kt0yJCLG1E7HvrbztSREJLj0s2q7mnbm7tQUbR+gYkZcb/t2e6+kuDho1qxxP/umm+zP3L0bvvqqcT9bROrkleUxbcM0AO7tf6/LP69NVBsC/AIoqigit0rdzN5OxYy43taj0yM7dWr8zw4Jgd/8xj7+5z8b//NFBIC3175NSWUJ3eK6MbTNUJd/XoBfAElRSQDsK9/n8s8Ts1TMiGtVVsKOHfZx585mMtxzD/j724vo1U4PF5FGU2PVMGWlPfD3vv73ndfu2PVRO25mf8X+Rvk8MUfFjLjWjh32yr9NmzbeLKafatPm2DTtKVPMZBDxYT/s+oHtOduJDI7k5u43N9rnJjdNBiCrIku/7byc/veKax3fxdRYs5hO5b6ji3O9+y7k55vLIeKDjt+HKSLYBXuynUaL8BYE+wdTaVWCob+lpHGomBHXqa4+tlCeqS6mWsOH2zOpiovtdWdEpFHsydvDV9vtwfeNMfD3eH4OP9pEtbG/aNuoHy2NTMWMuM7u3fbKv2Fh0Lq12SwOB9x79B/SKVNAUzVFGsW/V/6bGquGi5MvrtvRujHVDgImqdE/WhqRihlxndoupo4dXb8X07m49VYID7dzzZljOo2I1yurKuO1Na8BMP6C8UYyJDU9VszUWDVGMojrucFvGPFKlnVsFlPHjmaz1IqMhNtus481EFjE5T7a9BHZJdkkRiZyZeqVRjIkhCcQ4AiAENhZuNNIBnE9FTPiEsH5+ZCXZ0+JbtvWdJxjaruavvgC9mntCRFXqh34e3e/uwnwCzCSwd/Pn/hAe5+m1TmrjWQQ11MxIy4RuXevfZCUBEFBZsMcr2tXuOgie3Dyf/5jOo2I11qxfwU/7v+RIP8g7uhzh9Es8UF2MbMqZ5XRHOI6KmbEJaJqi5kOHcwGOZXaadqvvgoVFWaziHip2laZ67teT1xYnNEsLYNbArAmZ432afJSKmbE6UKA8MxM+wt3LGauuQZatoSDB+HTT02nEfE62SXZfLDxA8Be8de02MBYqIQjFUfYlrPNdBxxARUz4nTDAb/qaoiKgpgY03FOFhh4bL8mDQQWcbo31rxBeXU5fRL6MKDVANNx8Hf4w9EhcvN3zzcbRlxCxYw43WW1Bx06mF3190zuvBMCAmDRIli/3nQaEa9RVVPFlBX2Hwnj+49vtH2YzmqPfTd/j4oZb6RiRpzu8toDd+xiqtWyJYwZYx+//LLZLCJe5POtn7Mnfw8xoTHc2O1G03GOOVrMLMpYZDaHuISZuXLitYL37iUFsBwOHMnJpuOc2X33wccfw9Sp8Le/2Zthisg5y8jIIDs7+4TH/nfR/wJwdaur2bJhS73Ot2VL/V5fL/vs7qa9BXvZk7fn2GJ64hVUzIhTRS5ZAkBRfDwRwcGG05zFsGH2VO1Nm+Dtt+GBB0wnEvEYGRkZdOrcidKS0mMPtgLuBKrg9bte5/Wi1xt07qKiIqdkPEEldIrqxKa8TSzKWKRixsuomBGnily6FICCxEQab2/cBqrdr+m+++yBwL/9rXtsuyDiAbKzsyktKWXMY2OITYoF4Pvc79lVtovUiFSGPze83udMW57G3DfmUlZW5uS0tt7NerMpbxMLMxZyc4+bXfIZYoaKGXGeqirCV9srbBa0akUrw3HOya23wiOP2Lt7z5kDo0aZTiTiUWKTYklITSC/LJ/0zHQARnQZQXx4fL3PlZ2RffYXnYdezXoxdddUFmYsdOnnSOPTn6HiPKtX419cTC5Q2ry56TTnJiLi2H5NGggs0mA/7v8RC4u2Tds2qJBpDL2a9QJg8+HN5JTkmA0jTqViRpxn7lwA5oNnddfU7tf05ZeQkWE2i4gHqqiuYFWmvVXAwFYDDac5vejgaDrHdAZg8d7FhtOIM3nQbxxxe0eLmbmGY9Rbly4wYgTU1Gi/JpEGWJu1lvLqcpqFNCO1earpOGc0pM0QQFO0vY3RYmby5Mn079+fiIgI4uLiuOaaa9i27cSlpi3LYtKkSbRs2ZKQkBCGDx/Opk2bDCWW06qstBegwwOLGTjWOvPqq1BebjaLiAepsWpYtm8ZAANaDXCfRfJOY2iboQAaN+NljBYz8+fP57777mPZsmXMnj2bqqoqRo8eTXFxcd1rnn32WZ5//nleeuklVqxYQXx8PJdccgmFhYUGk8tJVqyA4mKqoqLYaDpLQ1x9tb2Q3uHD2q9JpB7Sy9LJLcslJCCEXvG9TMc5q9qWmZUHVlJSWWI4jTiL0WJm5syZ3H777XTt2pWePXvy5ptvkpGRwapVdt+rZVm88MILPP7444wdO5Zu3brx9ttvU1JSwnvvvWcyuvzU0S6mwn798Mg9aQMD4a677GMNBBY5Z+uK1wFwQasLCPIPMpzm7No2bUuriFZU1VTx4/4fTccRJ3Grqdn5+fkANGvWDID09HSysrIYPXp03WuCg4O56KKLWLJkCXfV/vI5Tnl5OeXHdRMUFBS4OLUAJxQz/PCDSz/KVauEBgwYQPeAABxLlsDatdCrl0s+R8RrJEN2ZTYBfgFc0OoC02nOicPhYEibIXy46UMW7lnI8LbDTUcSJ3CbYsayLCZOnMiQIUPo1q0bAFlZWQC0aNHihNe2aNGCPXv2nPI8kydP5qmnnnJtWDlReTkstmcGFPXr57KPySwqwgHccsstLvuMj/z9+QXYrTOvvuqyzxHxCnaPDb3jexMaGGo2Sz0MbTOUDzd9yKK9GgTsLdymmBk/fjzr169n0aKTv7l+OqDMsqzTDjJ79NFHmThxYt3XBQUFJCYmOjesnGj5cigrgxYtKHPhfkx5ZWVYwEsjRjAoJcXp599y+DD/mj7dLmamTYNnn4XoaKd/jog32Jq/FdqDAweDEwebjlMvteNmluxdQlVNFQF+bvOrUBrILf4P/va3v+XLL79kwYIFtG7duu7x+Hh74aWsrCwSEhLqHj906NBJrTW1goODCXb3PYG8zdEuJoYPt7cIcLEO0dH0Oe77wZkWAqXt2xOyc6e9X9OECS75HBFP986OdwBo16QdTZs0NRumnrrFdSMqOIr88nzWZa2jb8u+piPJeTI6ANiyLMaPH89nn33GnDlzSP7JX/XJycnEx8cze/bsuscqKiqYP38+gwd71l8CXq22mBkxwmwOJzl8/fX2wZQp9tozInKC9Nx0Zh+w/13uGd7TcJr68/fz58I2FwJab8ZbNKiYadeuHTk5Jy8FnZeXR7t27c75PPfddx9Tp07lvffeIyIigqysLLKysigttXdhdTgcTJgwgWeeeYbp06ezceNGbr/9dkJDQ7npppsaEl2crbQUjm4u6S3FzJErrrC3OUhLg+MKaRGxPbf0OWqogR0QExhjOk6DDEm0u5q03ox3aFAxs3v3bqqrq096vLy8nP3795/zeV555RXy8/MZPnw4CQkJdbcPP/yw7jUPPfQQEyZM4N5776Vfv37s37+f7777jogIt9+T2TcsWQIVFfYaLS4Yx2JCTWgo/OpX9hfPPWc2jIibOVx8mDfWvGF/4cE7AgxNshfPW5SxCMvyyAUl5Dj1GjPz5Zdf1h3PmjWLqKiouq+rq6v54YcfaNu27Tmf71y+gRwOB5MmTWLSpEn1iSqNpbaLaeTIRhkv02gmTLBnNM2eDWvWQO/ephOJuIWXfnyJ0qpSukR1YXP6ZtNxGqxfy34E+QdxsPggO47sIKW5d/wx5qvqVcxcc801gF1gjBs37oTnAgMDadu2Lc/pL1nf4mXjZeokJ8P118P778Pf/w5apFGEwvJC/vXjvwC4rcNtPMIjhhM1XJOAJlzQ6gIWZSxiYcZCFTMerl7dTDU1NdTU1NCmTRsOHTpU93VNTQ3l5eVs27aNK6+80lVZxd0UFcGPR1fQ9LZiBuDBB+37jz6C9HSzWUTcwCsrXyG3LJeOzTsyMmGk6TjnrXbcjAYBe74GjZlJT08nJsYzB32JEy1eDFVVkJRkt2R4m9694ZJLoLoa/u//TKcRMaq0spTnltot748OeRR/h7/hROevdtyMBgF7vgavM/PDDz/www8/1LXQHO+NN94472DiAby0i+n47RIixo4lZfZsav77XzZcfTXV57GIXkxMDG3atHFGRJFG99rq1zhUfIi2TdtyU/eb2LBug+lI521w4mAcONhxZAdZRVnEh8ebjiQN1KBi5qmnnuLPf/4z/fr1IyEhwe23fBcX8bJi5nTbJawE+paX89moUfz5PM4fGhLClq1bVdCIx6moruDZJc8C8PCFDxPoH2g4kXM0bdKU7i26s/7gehZlLOK6LteZjiQN1KBi5t///jdvvfUWt956q7PziKcoKICju5t7SzFzuu0SonfuhB9+4E/BwVxz881YAfX/sdly+DC3TJ9Odna2ihnxOO+se4d9BftICE/g9l63m47jVEPbDGX9wfUs3LNQxYwHa1AxU1FRoRV4fd3ChfZYkvbtwcv2vjppu4QWLWD1agJyc+mdmQkXeMbuwCLOUFVTxV8X/RWABwc/SJOAJoYTOdeQNkN4ecXL2nTSwzVoAPAdd9zBe5qq6tu8rIvpjPz8YNAg+3jpUruIE/ERH278kJ25O4kJjeE3fX9jOo7T1W46uTZrLQXlBYbTSEM1qGWmrKyM//73v3z//ff06NGDwMAT+0+ff/55p4QTNzZnjn3vC8UMQK9eMH8+5OXB+vVaRE98Qo1VwzOLngFgwoAJhAWFGU7kfK0jW9O2aVt25+1m2b5ljG4/2nQkaYAGFTPr16+nV69eAGzcuPGE5zQY2AccOQJr19rHvlLMBAbC4MH2isALF0LPnnaLjYgX+3zr52w+vJmo4CjGXzDedByXGdpmKLvzdrNwz0IVMx6qQcXM3NouBvFNCxaAZUHHjnD82BJv16+fvbZObq7dOnO0oBfxRpZl8fTCpwEYf8F4oppEneUdnmtom6G8u/5drTfjwfSnpdSfL42XOV5QkN06A3brzE/WVxLxJrN2zmJ15mpCA0OZMHCC6TguVTtuZvn+5VRUVxhOIw3RoJaZESNGnLE7aU7teArxTr5azAD072/vFH7kCGzYYHc3iXgZy7L483x7VaW7+95NTKh3r/jeKaYTzUOak1Oaw6oDqxiUOMh0JKmnBrXM9OrVi549e9bdunTpQkVFBatXr6Z79+7Oziju5PBh+5c4wPDhRqMYERR0bGbTggVqnRGvNGvnLJbuW0pIQAgPXvig6Tgu53A46lpn1NXkmRrUMvN/p9mnZtKkSRQVFZ1XIHFz8+fb9926QVyc2SymHN86s3Ej9OhhOpGI01iWxZPzngTgnn73+MwS/8OShvHFti+Yt3seD134kOk4Uk9OHTNzyy23aF8mb+fLXUy1goPVOiNe69u0b/lx/4+EBob61C/1i5MvBmDBngUaN+OBnFrMLF26lCZNvGt1SPkJFTO2Cy6AkBDIybFbZ0S8wPGtMvf1v48W4S0MJ2o83Vt0JzY0luLKYpbvW246jtRTg7qZxo4de8LXlmWRmZnJypUr+dOf/uSUYOKGsrJgyxZwOOCii0ynMSs4GAYOtIu7+fPtbjetOyMe7qvtX7EqcxVhgWE8ONj7x8ocz8/hx8XtLuaDjR/w/a7vGZo01HQkqYcG/esbFRV1wq1Zs2YMHz6cb7/9lieffNLZGcVdzJtn3/fsCc2aGY3iFgYMsFtnjhyx150R8WDHt8rcP+B+YsNiDSdqfKOSRwHwffr3hpNIfTWoZebNN990dg7xBL62hcHZBAfDhRfC99/bY2e6dwd/f9OpRBrk862fszZrLRFBEfx+0O9NxzFiVDu7mFm+bzkF5QVEBkcaTiTn6rzaxVetWsXUqVOZNm0aa9ascVYmcVcaL3Oy/v0hLMxeFbh2iwcRD1NVU8Uf5/4RsFtlmoc2N5zIjKSmSbSPbk+1Vc383fNNx5F6aFAxc+jQIUaOHEn//v25//77GT9+PH379uXiiy/m8OHDzs4o7mDfPtixwx4XMmyY6TTuIygIhtjrU7BgAVRVmc0j0gDvrHuHzYc30yykGX8Y/AfTcYyqbZ2ZvWu24SRSHw0qZn77299SUFDApk2bOHLkCLm5uWzcuJGCggLuv/9+Z2cUd1DbKtOnD0R57x4tDdK3L0REQEEBrF5tOo1IvZRWlvLE3CcAeHzo4zRt0tRsIMMu63AZYE9RtyzLcBo5Vw0aMzNz5ky+//57OnfuXPdYly5dePnllxk9WjuOeiV1MZ1eYKDdOjNjBixaBL1724+JuImMjAyys7NP+dxbO95if+F+4kPiGRQwiNXnWJBv2bLFmRHdxsXJFxPoF8jO3J2kHUkjtXmq6UhyDhpUzNTU1BB4in+sAwMDqdECYt5JxcyZ9elj76hdUACrVtnTtkXcQEZGBp06d6K0pPTkJ0OAB4AmkPVeFoMfHlzv83vbqu8RwREMSxrGD+k/8G3atypmPESDipmRI0fywAMP8P7779OyZUsA9u/fz+9+9zsuvvhipwYUN7B7t33z9z82PkROFBBgjyX6+mu7daZPH3s8jYhh2dnZlJaUMuaxMcQmnTjdelnBMtYXr6dZQDPG3jMWP8e5jzxIW57G3DfmUlZW5uzIxv0s5Wd1xYy37xjuLRpUzLz00ktcffXVtG3blsTERBwOBxkZGXTv3p2pU6c6O6OYVtsq07+/PTZETq1XL7t1JjcXVqywp22LuInYpFgSUhPqvs4vy2fTj5sAuKzTZbRq3qpe58vOOHW3lTe4IuUKJn43kfl75lNUUUR4ULjpSHIWDSpmEhMTWb16NbNnz2br1q1YlkWXLl0YNWqUs/OJO6gtZkaONJvD3fn7260zX3xhFzX9+tlr0Yi4oR/Sf6DaqqZtVFs6NOtgOo5bSW2eSrvoduzK3cUPu37g6k5Xm44kZ1Gv2Uxz5syhS5cuFBQUAHDJJZfw29/+lvvvv5/+/fvTtWtXFi7U9ulexbI0XqY+evSA5s2htBSWa38XcU978/ey4dAGAC5pfwkOh8NwIvficDj4WcrPAPhy25eG08i5qFcx88ILL3DnnXcSGXnyqohRUVHcddddPP/8804LJ25g5057jZnAQBhc/8GBPsfP79i+VUuXgheOJxDPZlkWM3bMAKBXfC9aRrQ0nMg9XdPpGgC+2PYFVTVaP8rd1auYWbduHZdddtlpnx89ejSrVq0671DiRmpbZQYOhNBQs1k8RdeuEBtrFzJLl5pOI3KCNVlryCzKJNg/mIuTNWHjdIYlDaN5SHNySnO0GrAHqFcxc/DgwVNOya4VEBCgFYC9jfZjqj8/Pxg+3D5etgxKSozGEalVVlXGnHT7Z3pY0jANbD2DAL+AutaZT7d8ajaMnFW9iplWrVqxYcOG0z6/fv16EhISTvu8eBiNl2m4zp2hRQuoqFDrjLiN+XvmU1xZTPOQ5gxoNcB0HLd3bedrAZi+dTo1ltZQc2f1KmauuOIKnnjiiVOuK1BaWsqTTz7JlVde6bRwYtimTXDwIISEwKBBptN4FofjWOvM8uUElJ5iwTKRRpRXlceP+38E4NL2l+Lvpx3ez+bidhcTFRxFVlEWS/YuMR1HzqBeU7P/+Mc/8tlnn5Gamsr48ePp2LEjDoeDLVu28PLLL1NdXc3jjz/uqqzS2H74wb4fMkRTjBuiY0do2RIOHKDFunWm04gHONO2Aw1Vu+3AkoIl1Fg1pDZLJaV5ilM/w1sF+Qfx844/59317/Lhxg8Z0kaLhrqrehUzLVq0YMmSJdxzzz08+uijdZtwORwOLr30UqZMmUKLFi1cElQMqC1mtKpzw9S2zrz3HrGbNhFvOo+4tTNuO3C+UmFf+T78HH6Mbq/98+rjpu438e76d3l/4/s8d+lzBPlrZW93VO9F85KSkvj222/Jzc1lx44dWJZFSkoK0dHRrsgnplRVwfyjI/hVzDRchw7QujV++/bxqOks4tbOtO3A+di2fBvzw+2f5YGtB9I8tLnTzu0LRrUbRUJ4AplFmXyb9m3doGBxLw1aARggOjqa/v37OzOLuJOVK+1NE5s2tXeBloZxOOzB0+++y13A9oMHTScSN/fTbQfO15J9S8AfgglmWJthTjuvrwjwC+Dm7jfzj6X/4J1176iYcVP1GgAsPqS2i2nECHuZfmm45GQKExIIBuJff910GvEhRRVFbPXbCkA3/24EB2jsW0OM6zUOgK+3f01OSY7hNHIqKmbk1GqLGe23df4cDjL79QOg+Rdf2DuQizSCH9J/oMpRBfuhjaON6Tgeq1tcN3rH96ayppJpG6aZjiOnoGJGTlZaCkuOTkPUeBmnKEpIYDbgV1UF//u/puOID9hfsJ+1WWvtL2ag/ZfO0697/xqAKSumaM0ZN2S0mFmwYAFXXXUVLVu2xOFw8Pnnn5/w/O23347D4TjhNnDgQDNhfcnixVBeDq1aQWqq6TRe40+1B2+/DTt2mIwiXs6yLGbunAlAYk0i7DMcyAvc1vM2IoIi2Jazje93fW86jvyE0WKmuLiYnj178tJLL532NZdddhmZmZl1t2+//bYRE/qo46dk6685p1kO5F94IVRXw5//bDqOeLGNhzayr2AfgX6BdKvuZjqOV4gIjuD2XrcD8K8f/2U2jJzEaDFz+eWX85e//IWxY8ee9jXBwcHEx8fX3Zo1a9aICX2U1pdxmQP33GMfTJsG27aZDSNeqaK6gtm7ZgMwtM1QQggxnMh7jL9gPADfbP+GXbm7DKeR47n9mJl58+YRFxdHamoqd955J4cOHTrj68vLyykoKDjhJvWQm2tPywYVMy5Q2rkzXHUV1NTAP/5hOo54oWX7llFYUUjTJk0ZlKhtSJwptXkql7a/FAuL55Y8ZzqOHMeti5nLL7+cadOmMWfOHJ577jlWrFjByJEjKS8vP+17Jk+eTFRUVN0tMTGxERN7gXnz7A0mO3a0x8yI8z3yiH3/zjuQmWk2i3iVksqSuj2ELk6+mAC/Bi8lJqfx8IUPA/DamtfYX7DfcBqp5dbFzA033MDPfvYzunXrxlVXXcWMGTPYvn0733zzzWnf8+ijj5Kfn19327t3byMm9gLqYnK9wYPt/a4qKuCFF0ynES+yeO9iyqvLaRHWgq6xXU3H8UrD2w5nSJshVFRX8OziZ03HkaPcupj5qYSEBJKSkkhLSzvta4KDg4mMjDzhJvWgYqZxPGz/dccrr0BentEo4h0KywvrdsUemTxSU7FdxOFw8ORFTwLw39X/JbNQravuwKOKmZycHPbu3UtCgvOW+pbj7N8PW7ce2yBRXOeKK6BrVygshH//23Qa8QILMhZQVVNFYmQiKc20K7YrXZx8MYNaD6Ksqoyn5j9lOo5guJgpKipi7dq1rF27FoD09HTWrl1LRkYGRUVF/OEPf2Dp0qXs3r2befPmcdVVVxETE8OYMWNMxvZec+bY9336gGaNuZafHzz0kH38wgtQVmY0jni23NJcVmeuBtQq0xgcDgd/G/U3AF5d/SrrD643nEiMFjMrV66kd+/e9D66keHEiRPp3bs3TzzxBP7+/mzYsIGrr76a1NRUxo0bR2pqKkuXLiUiIsJkbO81a5Z9P3q02Ry+4pe/hMREOHjQHgws0kCL9y6mxqqhXXQ72jZtazqOTxiaNJRfdPkFNVYNv5v1OyzLMh3Jpxkd6j58+PAzfgPMqv3lKq5XU3OsmLn0UrNZfEVgIEycCL/7Hfz97/DrX2tTT6m3wvLCum0LtCt24/rbqL/x5bYvmZM+h8+2fMa1Xa41HclnedSYGXGhNWsgOxsiImCQ1qZoNHfcAdHR9vYG06ebTiMeaPn+5VRb1bSObE2bKG0m2ZiSo5N5cPCDAIyfMZ7c0lzDiXyXihmx1bbKjBwJQUFms/iS8HAYb68qyt//bjaLeJyyqjJWHrAXuRySOERjZQx4fNjjdIrpRFZRFhO/m2g6js9SMSM2dTGZM368XUD++CMsW2Y6jXiQ1ZmrKa8uJzY0ltTm2hTWhCYBTXj956/jwMFba9/i2zTtH2iCihmBggJYYq8aqmLGgLg4uOkm+/jFF81mEY9RY9Ww4sAKAAa2HqhWGYMGJw7mgQEPAHD757dr7RkDVMyIPSW7qgpSUqBdO9NpfNMD9j+EfPKJvd6PyFnsOLKDvLI8mgQ0oXtcd9NxfN7kUZPp2aInh0sOc9vnt1Fj1ZiO5FNUzIi6mNxBr14wbJhdVE6ZYjqNeIDa1X57x/cm0D/QcBppEtCED677gNDAUL7f9b22OmhkKmZ8nWWpmHEXta0z//kPlJaazSJuLbskm525OwHo37K/4TRSq1NMJ/51+b8A+OOcP7J071LDiXyHihlft2MHpKfbA1C1hYFZV18NSUmQkwPvvWc6jbixNZlrAEhtlkp0SLThNHK8X/X6FTd2u5Fqq5obP72RI6VHTEfyCSpmfN2MGfb9kCH2NGExx9//2DTtF1+0W81EfqK6ppp1B9cB0Duht+E08lMOh4P/XPkf2ke3JyM/g1998SutDtwIVMz4um++se+vuMJsDrH9+tcQGgobNsC8eabTiBvacWQHxZXFhAWGaUNJNxUZHMnHv/iYIP8gvtz2JS8se8F0JK+nYsaXFRUd+4V55ZVGo8hR0dEwbpx9rGnacgprsuwuph4teuDvp+0v3FXvhN7836X/B8BD3z/E8n3LDSfybipmfNn330NFBbRvD6lacMtt3H+/ff/ll7Brl9ks4laKKorYnrMdsGcxiXu7p989XNflOqpqqrjhkxu03YELqZjxZV9/bd9feSVowS330amTPbPMsuyZTSJHbTy0EQuLVhGtiA2LNR1HzsLhcPDaVa/RLrode/L3cP/M+01H8lpGd80Wg2pqjo2XUReT+7n3XnvK/BtvwJ//DMHBphOJG9h0aBOAFslroC1btjj9nOXl5QSf5efzia5P8D+L/oep66fSPbA7o1qOOut5Y2JiaNNGG4eeKxUzvmr1asjKsmcwDRtmOo381BVXQOvWsG+fvSrwzTebTiSG5Zbmsq9wHw4cdI3rajqORyk6UgTALbfc4vyTO4Bzmaw0EhgGDy98GF4Gis/88pDQELZu2aqC5hypmPFVta0yo0drl2x3FBAAv/kNPPEEvPKKihlh4+GNALRt2pbwIC2jUB9lRWUAjLh3BCk9nTcDLG15GnPfmHtO5622qvk8+3NyQnPo8EQHRjYdedrXHt5zmOnPTCc7O1vFzDlSMeOrjh8vI+7pjjvsLqbFi+2p2t3VteDLNh6yi5lucd0MJ/Fc0a2iSUhNcNr5sjOy63XeMQljeG3Na+wo3cHglMEkRyc7LYuv0wBgX5SZCStX2seXX242i5xeQgJcc419/MorRqOIWYeLD3Oo+BB+Dj86x3Q2HUcaqFVkq7rtJ75J+4aqmirDibyHihlf9O239n3//hAfbzaLnNk999j3774LhYVms4gxW7Ltgavto9sTEhhiOI2cj5HJIwkPCienNKdus1A5fypmfNHnn9v3V11lNIacgxEj7DWAioq0X5MP25azDbA3MhTP1iSgCSPb2uNlFmYspLRSm8o6g4oZX1NYCLNn28djxpjNImfncMDdd9vHr7yi/Zp8UEF5AQcKDwCQ2lyLW3qDnvE9iQuLo6yqjAUZC0zH8QoqZnzNzJlQXg4dOkBXTe/0COPGQZMmsG4dLFtmOo00sm3ZdqtMYmSiZjF5CT+HH5e0uwSAFftXkFeWZzaQF1Ax42umT7fvx4zRqr+eolkzuPFG+1gDgX3O1pytAHSM6Wg4iThT++j2JDdNptqqZvHexabjeDxNzfYlFRV168tkDRzIgdWrnf4RrlhhU7AHAr/1Fnz8sb0BZXS06UTSCMqqytidtxuATs01XsabOBwOhiUNIz0vnTWZaxjaZiiRwZGmY3ksFTO+ZO5cKCigOjaWDjffTHFZmcs+qrCoyGXn9kn9+0OPHrB+PUybBuPHm04kjSDtSBo1Vg0xoTE0D21uOo44WdumbWkT1YaM/AwW713M5R20VEZDqZjxJUe7mI4MG0bxp58ydcwYOsc6d7O6b9PS+NPcuZS5sFDySQ6HvYje/ffDq6/Cffepm9AH1I6XUauM9xrWZhhTN0xldeZqhrUZRlhQmOlIHknFjK+oqYEvvgAgb/hw+PRTOsfG0ifBeathAmzJznbq+eQ4t9wCDz5ot86sXGm31ojXqqqpIu1IGqAp2d6sXXQ7WkW0Yn/hflYeWMlFbS8yHckjaQCwr1i2zN5YMjKSIv0S9EzR0XDddfbxq6+azSIutydvDxXVFYQHhdMyoqXpOOIiDoeDga0HArDiwAqtCtxAKmZ8xWef2fc/+xlWYKDZLNJwd95p37//vr2QnnitHbk7AOjQrAMOdSl6tc4xnYkIiqC4sphNhzaZjuORVMz4gpoa+Ogj+/gXvzCbRc7PsGGQkmIXMh9+aDqNuNDOIzsBu5gR7+bv588FrS4AYNn+ZVhaHLPeVMz4guXLYe9eCA/XxpKernYgMMBrr5nNIi6TX5bP4ZLDOHDQrmk703GkEfRJ6EOAXwBZRVkcrDxoOo7HUTHjC2r/gr/6anslWfFs48ZBQIA9DmrjRtNpxAVqu5haR7bWxpI+IjQwlG6x3QDYWrLVcBrPo2LG29XU2AutAVx/vdks4hwtWsDPf24fq3XGK9V2MbVv1t5wEmlMfRL6ALCzdCcEGw7jYTQ129stXgwHDkBkJFx6qek0Ps9ZKyRHjhhBh88+o+qtt9hwww00b9WKNm3aOOXcYlaNVcOu3F0AdIjWeBlf0jqyNbGhsRwuOQzdTafxLCpmvF3twN9rroFglfqmZBYV4QBuueUWp5zPD9gFJOXn8+zgwXwZEsKWrVtV0HiBg5UHKa8uJzQwVFOyfYzD4aBPQh9m7ZwFfU2n8SwqZrxZdTV88ol9fMMNZrP4uLyyMizgpREjGJSS4pRzBq5aBatW8X+xsXxw+DDZ2dkqZrzA3rK9gL0RoaZk+54eLXrw/c7vqU6oZnv+dvrQx3Qkj6BixpstXGgvlBcdDaNGmU4jQIfoaOetujx0KKxaRfzhw2i+i/fYV74P0HgZXxUaGEqbJm1IL0tnxv4Z3MiNpiN5BA0A9mbvv2/fX3MNBAUZjSIuEBUFHewxFb82HEWcJAyyq+wtQdpHq5jxVR1C7J/rWftnUWPVGE7jGVTMeKuysmPjZW6+2WwWcZ0+dhP0rwAqK41GESdItu/iw+MJDwo3m0WMaRPcBsrgYNlBFuxZYDqOR1Ax462+/Rby8qBVKxg+3HQacZXUVCpDQkgAohYvNp1GztfRYia5abLZHGKUv8MfNtvH09ZPMxvGQ6iY8Vbvvmvf33QT+PubzSKu4+9PTmoqADGff242i5y/o4OfVMwI6+27T7Z8QnlVudksHsBoMbNgwQKuuuoqWrZsicPh4POf/GNsWRaTJk2iZcuWhISEMHz4cDZt0iZcZ3XkCHzzjX18661ms4jL5XTsCEDk4sWwf7/hNNJQ+0v2QzQ4cNAmSrPSfN4eiGsSR15ZHt+mfWs6jdszWswUFxfTs2dPXnrppVM+/+yzz/L888/z0ksvsWLFCuLj47nkkksoLCxs5KQe5qOP7PETPXtCd6285O3KmzZlPuCoqYG33jIdRxpoRfYKAOIC4wgO0JpQPs+CS1vZC51O26CuprMxWsxcfvnl/OUvf2Hs2LEnPWdZFi+88AKPP/44Y8eOpVu3brz99tuUlJTw3nvvGUjrQaZOte+dtECbuL+6TQ1ef93ewkI8Tm0x0yq4leEk4i4ub2VvDPz19q/JL8s3nMa9ue06M+np6WRlZTF69Oi6x4KDg7noootYsmQJd9111ynfV15eTnn5sf7FgoICl2d1lYyMDLKzs+v1nqB9++i2eDGWw8HG7t2pXL36pNc4a0l9cR+fAm+GhxOQng5z58LFF5uOJPVgWVZdMdMySKv+ii01MpWOzTuyLWcb36R9w03dbzIdyW25bTGTlZUFQIsWLU54vEWLFuzZs+e075s8eTJPPfWUS7M1hoyMDDp36kRJaWm93vcnoBvwvWUx+rLLzvjawqKihgcUt1IK5F5+ObEff2xvPqlixqNsyd5CTnkOVEJcUJzpOOImHA4H13a+lmcWPcNnWz5TMXMGblvM1Prpct6WZZ1xie9HH32UiRMn1n1dUFBAYmKiy/K5SnZ2NiWlpUwdM4bOsbHn9ibLousHH0BhISnDh7Pq6CyXn/o2LY0/zZ1LWVmZExOLadnXXGMXM599Bjk50Ly56Uhyjuakz7EPMiCgjdv/syyNaGznsTyz6Blm7JhBSWUJoYGhpiO5Jbf9qYmPjwfsFpqE45Z/P3To0EmtNccLDg4m2Is2VOwcG3vuy9/v2gWFhRAcTNvBg2kbGHjKl22pZ9eVeIbSTp3sRfRWr7bHTT3wgOlIco5+SP/BPkgHhhiNIm6mT0If2jZty+683czcMZOxnU8eYypuvM5McnIy8fHxzJ49u+6xiooK5s+fz+DBgw0mc2Nr19r33brBaQoZ8XJ33GHfv/YaWJbZLHJOqmuqmbd7nv1FutEo4oYcDgdjO9kFzGdbPjOcxn0ZLWaKiopYu3Yta4/+Ek5PT2ft2rVkZGTgcDiYMGECzzzzDNOnT2fjxo3cfvvthIaGctNN6jc8SWkpbD66ZGQf7bLqs375SwgJgY0b4ccfTaeRc7Amaw15ZXmEBYRBpuk04o6u7XItAF9t/0oL6J2G0WJm5cqV9O7dm969ewMwceJEevfuzRNPPAHAQw89xIQJE7j33nvp168f+/fv57vvviMiIsJkbPe0cSNUV0NcHDhrV2bxPE2bwi9+YR+/9toZXyruoXa8TL/m/UCz6uUUBrYeSEJ4AgXlBce6JOUERouZ4cOHY1nWSbe3ji785XA4mDRpEpmZmZSVlTF//ny6detmMrL7WrPGvu/dG84wQFp8wK+P7qH9/vv2GCpxa7W/nPrF9DOcRNyVn8OPMZ3GAOpqOh23HTMj9ZCVBZmZ9h5MPXqYTiOmDR0KKSlQXHxs53RxSxXVFSzcsxCAC2IuMJxG3FltV9PnWz+nqqbKcBr3o2LGG9QujNexI4Rq2p7PczhOHAgsbmv5vuWUVpUSGxpL+4j2puOIGxuWNIxmIc3IKc1h6d6lpuO4Hbedmi3nqKIC1h/dXlUDf33a8Ss7B/TqRXd/fxzLlrH5o48o69ChweeNiYmhTRttfOgKtV1MI5NHnnH9LJEAvwCuSLmCqeun8uW2LxmaNNR0JLeiYsbTbdwI5eXQrBm0a2c6jRiQWVSEA7jlJ3txfQqMBWbdcAMTT/XGcxQaEsKWrVtV0LhA7eDfi5O1YrOc3VWpVzF1/VS+2v4Vfx/9d9Nx3IqKGU9mWbBypX3ct68G/vqovLIyLOClESMYlJJS93hkRgbMnMlvg4MZfsstWP7+9T73lsOHuWX6dLKzs1XMOFlxRTHL9i0D7JaZ/N3aSFDO7NL2lxLgF8C2nG2k5aSR0jzl7G/yESpmPNmBA8cG/vbqZTqNGNYhOvrE1aJbtIDFiwkoLKR3Xp69mKK4jUUZi6isqSQpKol20e1Ys3uN6Uji5qKaRDG87XC+3/U9X23/iomDzqfN1btoALAnq22V6dpVA3/lZH5+9lR9ODZ1X9yGxstIQ1yVehUAX2770nAS96JixlOVltrjZQD6aX0KOY3aYmbXLsjNNZtFTqDxMtIQtcXMooxF5JbqZ7qWihlPtW4dVFXZK/62bm06jbirpk2PDQxX64zbyC3NZXWmvaTCiOQRhtOIJ0mOTqZbXDeqrWpm7JhhOo7bUDHjiSwLVqywj/v108BfObPaKftr10KN1st3B3N3z8XColNMJ1pGtDQdRzxMbevMV9u/MpzEfaiY8URpaXDkCDRpAj17mk4j7q5jR3vzycJC2LHDdBpBXUxyfmqLmRlpM6isrjScxj2omPFEy+zpnPTuDUFBZrOI+wsIOFb0qqvJLdQO/lUxIw1xQasLiAuLI788n4UZC03HcQuamu1pDh2C9HS7a+kC7eUi56h3b7sI3rYNioogPNx0Ip+1v2A/W7O34ufwY3jb4abjiBs7flXvnxrYbCBfFn/J6wtfp2lu03M+p7eu6K1ixtPUtsp06mQP7hQ5F7UDxffts8fODBliOpHPqu1i6pPQh+iQaMNpxB0VHSkCTl7V+wSdgBvhvVXv8d6498753CGhIWzd4n0requY8SQlJbBhg308cKDZLOJ5+vSxi5lVq+DCCzVw3BB1McnZlBWVATDi3hGk9Dz1Kr+VNZW8c/AdqptV84uXfkF04NkL48N7DjP9Ge9c0VvFjCdZtcqejp2QAImJptOIp+nWDb77DvLyYOdOOI/NJ6VhLMuqa5kZmTzScBpxd9GtoklITTjt88mVyew4soMjEUfo0qZLIyZzPxoA7Cmqq49Nxx4wQH9VS/0FBh4bCFy7erQ0qh1HdrC3YC9B/kEMaaOuPjk/qc1TAdies91wEvNUzHiKzZvtqbXh4fb2BSINUbta9PbtkK+NDRtbbRfToNaDCA3UFiRyfjo27wjA3oK9FFcUG05jlooZT7F8uX3fr5891VakIWJioG1be+HF1atNp/E5Gi8jzhQZHEl8eDwAaUfSDKcxS8WMJ8jIgP377d2xtQ+TnK/a76HVq+3uS2kUNVYNc9PnAnBxOxUz4hzqarKpmPEEixfb9z16QFiY2Szi+Tp1sr+PiorsdWekUaw/uJ6c0hzCg8Lp37K/6TjiJWq7mnbm7qSqpspwGnNUzLi5JkeO2OMbwJ5OK3K+/P2P7aatgcCN5odddhfTsKRhBPoHGk4j3iIhPIGIoAgqqivYnbfbdBxjVMy4uRbr19sHnTtD8+Zmw4j36NvXvk9Ph5wcs1l8hMbLiCs4HA5Smttr0WzL8d2WVhUzbqw10Czt6KAutcqIMzVtCql2X7taZ1yvorqCBXsWACpmxPlqu5q252zHsizDacxQMePGJgIOy7Jnn7RqZTqOeJva1pm1a6FSO++60tK9SymuLCY2NJbuLbqbjiNeJrlpMgF+ARSUF3Cw+KDpOEaomHFT/vn53Fn7hVplxBU6dLBbaMrKjm2TIS4xa+csAEa3H42fQ//sinMF+gfSPro94LtdTfqpclOxH31EOFDSvDm0b286jngjPz/of3RWzfLl9toz4hK1xcyl7S81nES8la9P0VYx445KSoj94AMADvbsqa0LxHV697a3OTh0CHbvNp3GKx0qPsTqTHuBwtHtRxtOI96qtpg5UHiAwvJCw2kan4oZd/TGGwTm5bELyG3XznQa8WYhIcf2a/rxR7NZvNR3O78DoFd8L1qEtzCcRrxVeFA4rSLssZW+2DqjYsbdVFTA3/8OwHNgdwWIuNIFF9j327ZBbq7ZLF5IXUzSWHy5q0m/Kd3N229DRgYVMTG8bjqL+IbYWHtclmUd25ldnKLGqqlrmVExI65WO0V7V94uKqt9a4aiihl3UlkJzzwDwMFx4yg3HEd8SG3rzOrVduugOMW6rHUcKj5EWGAYF7bRrERxrbiwOKKCo6iqqWJX7i7TcRqVihl38s479iDMFi3IHjvWdBrxJSkp0KwZlJfDunWm03iN2i6mkckjCfIPMpxGvJ3D4ajravK1KdoqZtxFZSU8/bR9/NBDWE2amM0jvsXhONY68+OPmqbtJBovI43t+NWAa6waw2kaj4oZdzFtmr1PTmws3HWX6TTii3r1gqAgyM6GXb7VRO0KBeUFLM6wd7y/tIOKGWkcbZu2Jdg/mOLKYvYV7DMdp9GomHEHVVXHWmUefBDCwszmEd8UHGwXNADLlhmN4g1m7ZhFZU0lqc1T6dCsg+k44iP8/fzpGGO3zmw+vNlwmsajYsYdvP8+7NgBMTFwzz2m04gvGzjQ7nLasQMO+uYeL87y1favAPh56s8NJxFf0zmmMwBbs7f6zMaTKmZMq66Gv/zFPv797yE83Gwe8W3R0dCli328ZInZLB6sqqaKb9K+AeCqjlcZTiO+pn10ewL9Askvz+dA4QHTcRqFihnT3nsPtm+3Z5Lcd5/pNCIweLB9v2EDQYW+tyy6MyzZu4QjpUdoFtKMwYmDTccRHxPoH0hK8xQAtmRvMZymcaiYMamiAp580j5+8EGIiDCbRwSgZUtITgbLIla7aTfIV9vsLqYrUq4gwC/AcBrxRbVdTVuyt/hEV5NbFzOTJk3C4XCccIuPjzcdy3lee82ewRQfD7/9rek0IsdcaC/wFrN1K9GGo3iiL7d/CWi8jJiT0iwFf4c/R0qPcKj4kOk4LufWxQxA165dyczMrLtt8Ja/FEtK4H//1z7+4x81g0ncS7t2EB+Pf1UV95rO4mG2ZW9je852Av0CNSVbjAkOCK6bRecLXU1uX8wEBAQQHx9fd4uNjTUdyTleegmysqBtW7jzTtNpRE7kcNSNnbkfcJSVmc3jQWpnMQ1vO5zI4EjDacSXdYrpBPjGFG23L2bS0tJo2bIlycnJ3HjjjezyhsW88vLgr3+1j596yl6oTMTddO1KeXg4cUDzr782ncZjfLntaBdTR3UxiVmdYjrh7/DncMlhDhZ591ILbl3MDBgwgHfeeYdZs2bx6quvkpWVxeDBg8nJyTnte8rLyykoKDjh5naeew5yc+0psDffbDqNyKn5+XGoRw8A4t59115GQM7oYNFBFu+1V/29KlVTssWsJgFN6mY1bTjkJUM0TsOti5nLL7+ca6+9lu7duzNq1Ci++cZet+Htt98+7XsmT55MVFRU3S0xMbGx4p6bgwfh//7PPv7LX8Df32wekTPI6diRbKDJvn3w0Uem47i9z7Z8Ro1VQ7+W/UhqmmQ6jgjd47oDsPHQRq+e1eTWxcxPhYWF0b17d9LS0k77mkcffZT8/Py62969exsx4TmYPBmKi6FfP7jmGtNpRM6oJjCQF2q/+POf1TpzFh9v/hiAX3T5heEkIraUZikE+QeRX57PwUrv7WryqGKmvLycLVu2kJCQcNrXBAcHExkZecLNbezYAVOm2MfPPGMPshRxc/8CqiIiYOtW+Phj03Hc1qHiQ8zfMx9QMSPuI9A/kC4x9qreO0p3GE7jOm5dzPzhD39g/vz5pKens3z5cq677joKCgoYN26c6WgN8/DDUFkJl10Gl1xiOo3IOSkADt10k/3F//4v1NQYzeOuju9iSo5ONh1HpE63uG4A7Czd6ea/9RvOrf+z9u3bxy9/+Us6duzI2LFjCQoKYtmyZSQleWBf9IIF8Nln4OcH//iH6TQi9XL4l7+EqCjYvBk++cR0HLekLiZxV8nRyYQFhlFulUN702lcw62LmQ8++IADBw5QUVHB/v37+fTTT+lSuwmeJ6mpgYkT7ePf/Aa6djWbR6SeqiMi4He/s7/485/VOvMTmYWZzNs9D1AxI+7Hz+FH17ijv3d6mM3iKm5dzHiNadNg1Sp776WnnjKdRqRhHnjAbp3ZtEkzm37ivQ3vUWPVcGHihepiErfUI+5oFdMJCiu9bwNZFTOuVlICjz5qHz/+OMTFmc0j0lBNm8Lvf28f/+lP9vgvAeDd9e8CcEuPWwwnETm1lhEtiQ6IhkCYsX+G6ThOp2LG1Z57Dvbvt7cteOAB02lEzs/vfgexsfbMvDfeMJ3GLWw4uIF1B9cR5B/E9V2vNx1H5JQcDgedQu3tDb7I+MJwGufT3vTnKSMjg+zs7FM+F3j4MF2eeQZ/YNddd5G3+dz3x9iyxfs3BhMPFB5ut8rcf7/dZXrrrRAaajpVozv+5/7FzS8CcGHshezespvd7G7QOfUzL66WEpLC0iNL2Zq/lTWZa+id0Nt0JKdRMXMeMjIy6NypEyWlpad8firQHVgCXPjoo8e6m+qhsKjovDKKON1vfgPPPw+7d8O//mUvOeBDMjIy6NS5E6UlpXbb9gQgEua+OJe+W/qe9/mL9DMvLtLErwlsBbrBa6tf4+WfvWw6ktOomDkP2dnZlJSWMnXMGDr/ZDfv8AMHSP36aywgeswYVtVzt+9v09L409y5lGm3YnE3wcH2jKbbbrM3TL3zTmjWzHSqRpOdnU1pSSljHhtDcYtivsv9jiZ+Tbj5gZvxdzR8e5K05WnMfUM/8+Jiq4Fu9jivv476KxHBEaYTOYWKGSfoHBtLn+NXJa6uhunTAXD060fnHvWfC7flNF1XIm7hppvg2Wdh40a7sHnhBdOJGl1sUiwbyzcC0Ltlb1q3b31e58vO0M+8NIJ0SApLYk/xHt5Z9w73XXCf6UROoQHArrB8ORw+bI8lGDnSdBoR5/P3t7uaAF5+2d7qwMcUVRex44i9PHzfhPPvXhJpFBZcn2wPVH9pxUtes/mkihlnKyiA+fb+LIwaBSEhZvOIuMoll8BVV0FV1bFFIX3I1pKtWFi0jWpL89DmpuOInLMrW19JeFA4W7O38v2u703HcQoVM85kWTBjBlRUQOvW0KuX6UQirvXccxAYaH/ff/ut6TSNxw+2lWwDoE/LPobDiNRPeGA4t/e8HYAXl79oNoyTqJhxpi1b7OZ2Pz+48krtii3eLyXFnqYNduuMryyk1wWKa4oJCwyjc0xn02lE6u3+AffjwME3ad+w8dBG03HOm4oZZykttf86BRgyBFq0MJtHpLH86U/2Qnrbth0bR+PFLMuCQfZx/5b9CfDTPArxPCnNU7iuy3UA/HXRXw2nOX8qZpzlu++gqAhiYmDoUNNpRBpPVBT8/e/28VNPwa5dZvO42Prc9dAK/PGnX8t+puOINNijQ+y1zz7Y+AG7cj3751Z/UjhBxL59sHat/cVVV0GALqv4mNtug3fegTlz4J57YOZMr+1mnbZrGgAdQjoQFhTmks/IP5hPSX6JU86Vm5lr3+/PJXN7JqFRoUS1iHLKucWz9U7ozaXtL2XWzlk8u/hZ/n3lv01HajD91j1PTYGk2tlL/ftDmzYm44iY4XDAv/8N3bvbrZTvv2+vReNlCssLWZG9AoDuYd1d8hkleSVMGfcSFeVVTj3v3ClzmctcgoIDuPft8SpoBIDHhz7OrJ2zeH3N6zw4+EHaN2tvOlKDqJg5Ty8BQcXF9gqoo0aZjiNiTkqKPX7mj3+ECRPg0kuhuXdNWY4IjuCri7/iov+5iGYPuGbV4/LicirKq3hsaCeSos5/36sj+4+QvmY3yf3bUtikCc8s3EpJfomKGQFgaNLQutaZJ+c9ydSxU01HahAVM+ch+rvvuBmwHA4cY8ZAUJDpSCJmPfig3SqzaZPd3fThh17X3RQeGA7nvmdsgyVFhZLa/PyXmj+YX0IZkBzRhHwf3BRUzu6Zi59h1s5ZvLfhPR668CF6tKj/qvWmaQBwQ+3fT+LkyQBk9e5trysj4uuCguCtt+xxYx9/DFM98688EV/SJ6EP13e9HguLh2Y/5JGrAquYaaiHHyagoIAVQGYfLZolUqdfP5g0yT6+7z57d20RcWtPj3yaIP8gZu2cxRfbvjAdp95UzDTUCy9w5LLLuBXsRfJE5JhHHoELL4TCQrj1VnvzVRFxWx2adeAPg/4AwISZEyipdM5susai38INFRPD7qefZpvpHCLuyN8f3n0XwsNh0aJjLTUi4rYeG/oYiZGJ7Mnfw9MLnjYdp15UzIiIayQn29O1Af7yF/j8c6NxROTMwoLCePEye6+mvy3+Gyv2rzCc6NypmBER17n55mN7N912m713mYi4rTGdx3Bjtxuptqq57fPbKK0sNR3pnKiYERHX+sc/YNgwe/zMmDFQUGA6kYicwctXvEx8eDxbs7fy4OwHTcc5J1pnRkTOasuWLef1/oA//pFON99M0NatFIwaxc5//hPr6LpMMTExtNHK2SKN5lx+nh/r8hj3/3g/L694mfiqeK5ofcVpX+sOP8MqZkTktDKLinAAt9xyy3mfqw8wD4hcsYIdgwZxI1ADhIaEsGXrVuP/GIp4u6IjRUA9fp5HABfBn378E3/6zZ/g4KlfFhIawtYtZn+GVcyIyGnllZVhAS+NGMGglJTzPl/W/v2EzZjBL2pqGN65M9917Mgtn39Odna2ipmzcNRYOCwLaixwQI2/n9etriyuVVZUBsCIe0eQ0vPsP881Vg0zc2eyj32E3RfG1TFXE+4ffsJrDu85zPRnphv/GVYxIyJn1SE6mj4JCed/ooQECAmBTz4hdssWLvH3R7+Oj7Is2gPDdh7iQmDE6nTiKqsIKq0kqKwC/6qaE15e4+egMjiQipAgSiNDKIkMoaRpGPmxkZSHBavQkdOKbhVNQuq5/TzfVHkTb6x9g+ySbGYXzeZXvX5FSGCIixPWn4oZEWlcXbtCWRl8/TVxGzfyNkBlpelUjc+yiNt9mJRlaSSt20P8+j1EAszaZD9/IPeMb/ersQgurSC4tIKIo90HtcpCg8lvEcWRVs3IrdKChdJwIYEh3NL9Fl5f8zqHSw7z7vp3uaXHLYQGutc+XypmRKTx9e0LgYFYn3/OrZZF3kMPwYwZ4OUbIQaWVpC8djcpy7aTsnwHTQ/mn/B8GbA7LoJlhwrp3KklYfFNqQgJorJJEJXBAVgOh72xrWURWFFFQHklwSUVhBSUEJpfSviRIsKPFNGkpJwm6YdokX6IjkBvIHt/LpuStBmu1F9Ukyhu6XELb697m8yiTN5e9za39riV8KDws7+5kaiYEREzevRgZ0kJrWbNoumCBTB4sL05pRPG5riT6P1HSFmeRsryHSSvSSeg8lhLSWVQAOm9k9nZrx1zS8p54c15DLmuH3OnzOU/HeLPuGt2dVAAhDehqDlA87rH/SqricwuIDozj+b7cgjPLaYt0HbbQXqnHSICyNi8j7xz7GYQAYgLi+P2nrfzzvp3OFR8iNdWv8Yvu/3SdKw6KmZExJiCpCRuB+Y2a0bgunX2JpVvvgljx5qO1mD+ldW0Wb+H1GVppCxPI2ZvzgnP57WIYvvAFNIGprK7V1sqmwQCsPP7DTijs60m0J+8hGjyEqJJ75NM4ea9BK/YxQXhwTQtKmcsMGPbAZY74bPEt8SGxfKrXr9i2oZpHCk9whtr3+CiiItMxwJUzIiIYYuBrdOm0f3pp+19nK69FsaNg7//HWJjTcc7JxGHC0j5cQcpy9Not3IXwaUVdc9V+/uR0aMNaQNSSBuQwuGkmEYdnFvSJIhVQO4FyfhXQfqCrRQOStWKqdIgzUKacUfvO/ho80fsztvN7LzZcAWUV5cbzaViRkSMq4yLgzlz4LHH4Lnn4O234csvYfJkuOMOe+NKNxIGdNuQQf/v1tFu5S7i9hw+4fnCZuGkDehA2oAUdvVtR3l4EzNBf+JIZCgPAb+Jb4o6maShagcFz9k9hyV7l8AF8Njqx5jbf66xTCpmRMQ9BAbarTHXXQd33w1r19r3//gHPPSQvbdTcHC9TpmRkUF2dvZ5R/MrLSV040bC162j1dy5HAGCXppZ93yNw8HutrFs6N6GDd3bsK91cyy/o60vZ5mVVCs3035d4eHC884r4mr+fv5c0u4SooqimLFnBr8a8iujeVTMiIh7GTAAVqyAKVPgqadgxw74zW/giSfsgub666FPn7N21WRkZNC5UydKSuu3UZ4/0Al7BlA/4EKgByf/Y5kOfHf0NseyyEs/BOmH4MuV9fq8n1r5sf3+ipKK48f1irilxCaJ8CJ0u7mb0RwqZkTE/QQE2Ltt/8//wGuv2V1P+/bBs8/at/btYdQoGDTIvrVvf1JXVHZ2NiWlpUwdM4bOpxh741dZSXBBAcH5+XX3IUeOEHLkCH7VJ6/NUhEWRlF8PFkRETyzdi3TgIm92nJp62Zc6oT/5CP7j5C+Zjd5yTF8lJ5NVWWVE84q0gjcYCkjFTMi4r7Cw2HCBLj3XvjqK/jwQ/j6a9i507795z/26wIDoU0baNsWmjWD8HBal5TwL2D01q3EpqdDaSmUlEBxsX0rP8OAxaAgiI+3b4mJkJhIUFQUzYDyzExK164FIDGiyRmnT9fHwfwSyoCgkECnnE/El6iYERH3FxRkz3K69looKoLZs2HJEli6FFautAuT2gLnqDhgPMCZdggOCbGLn+bNIToa4uLsAiY6WtsBiHgQFTMi4lnCw2HMGPsGUF0NBw5Aejrs3g0FBVBYSNaOHbz6xhv8um9fWkZF2YVLaCiEhdm38HBo4h6zjETk/KiYERHP5u9f1xXEsGF1Dx9YvZon3niDn/XtS0tnbJIpIm7LI9ZNmjJlCsnJyTRp0oS+ffuycOFC05FERETETbh9MfPhhx8yYcIEHn/8cdasWcPQoUO5/PLLycjIMB1NRERE3IDbFzPPP/88v/71r7njjjvo3LkzL7zwAomJibzyyiumo4mIiIgbcOsxMxUVFaxatYpHHnnkhMdHjx7NkiVLTvme8vJyyo+bcpmfnw9AQUGB0/MVFRUBsOrAAYoqKs7y6vrZctheHn3D4cOE7Nmjc7v43K4+v859atuOrs67atWqup8np5172zb73E7++czOyaF284ItmbmUVTlnkY38QwVkATlHSpx67trzlu/LpTDYPndmWiYVped3TQ4f3cLhcPph9oQ553vDFefUeV173uy99s9wUVGR03/P1p7Psqyzv9hyY/v377cAa/HixSc8/vTTT1upqamnfM+TTz5pAbrppptuuummmxfc9u7de9Z6wa1bZmo5frLeg2VZJz1W69FHH2XixIl1X9fU1HDkyBGaN29+2vccr6CggMTERPbu3UtkZOT5BfdxupbOo2vpPLqWzqNr6Ty6liezLIvCwkJatmx51te6dTETExODv78/WVlZJzx+6NAhWrRoccr3BAcHE/yTzeiaNm1a78+OjIzUN5ST6Fo6j66l8+haOo+upfPoWp4oKirqnF7n1gOAg4KC6Nu3L7Nnzz7h8dmzZzN48GBDqURERMSduHXLDMDEiRO59dZb6devH4MGDeK///0vGRkZ3H333aajiYiIiBtw+2LmhhtuICcnhz//+c9kZmbSrVs3vv32W5KSklzyecHBwTz55JMndVVJ/elaOo+upfPoWjqPrqXz6FqeH4dlncucJxERERH35NZjZkRERETORsWMiIiIeDQVMyIiIuLRVMyIiIiIR/P5YiY3N5dbb72VqKgooqKiuPXWW8nLyzvt6ysrK3n44Yfp3r07YWFhtGzZkttuu40DBw40Xmg3MmXKFJKTk2nSpAl9+/Zl4cKFZ3z9/Pnz6du3L02aNKFdu3b8+9//bqSk7q8+1/Kzzz7jkksuITY2lsjISAYNGsSsWbMaMa17q+/3Za3FixcTEBBAr169XBvQg9T3WpaXl/P444+TlJREcHAw7du354033miktO6tvtdy2rRp9OzZk9DQUBISEvjVr35FTk5OI6X1MOe9gZKHu+yyy6xu3bpZS5YssZYsWWJ169bNuvLKK0/7+ry8PGvUqFHWhx9+aG3dutVaunSpNWDAAKtv376NmNo9fPDBB1ZgYKD16quvWps3b7YeeOABKywszNqzZ88pX79r1y4rNDTUeuCBB6zNmzdbr776qhUYGGh98sknjZzc/dT3Wj7wwAPW3/72N+vHH3+0tm/fbj366KNWYGCgtXr16kZO7n7qey1r5eXlWe3atbNGjx5t9ezZs3HCurmGXMuf//zn1oABA6zZs2db6enp1vLly0/aX88X1fdaLly40PLz87NefPFFa9euXdbChQutrl27Wtdcc00jJ/cMPl3MbN682QKsZcuW1T22dOlSC7C2bt16zuf58ccfLeCs/1h6mwsuuMC6++67T3isU6dO1iOPPHLK1z/00ENWp06dTnjsrrvusgYOHOiyjJ6ivtfyVLp06WI99dRTzo7mcRp6LW+44Qbrj3/8o/Xkk0+qmDmqvtdyxowZVlRUlJWTk9MY8TxKfa/l3//+d6tdu3YnPPbPf/7Tat26tcsyejKf7mZaunQpUVFRDBgwoO6xgQMHEhUVxZIlS875PPn5+TgcjgbtAeWpKioqWLVqFaNHjz7h8dGjR5/22i1duvSk11966aWsXLmSyspKl2V1dw25lj9VU1NDYWEhzZo1c0VEj9HQa/nmm2+yc+dOnnzySVdH9BgNuZZffvkl/fr149lnn6VVq1akpqbyhz/8gdLS0saI7LYaci0HDx7Mvn37+Pbbb7Esi4MHD/LJJ5/ws5/9rDEiexy3XwHYlbKysoiLizvp8bi4uJM2tzydsrIyHnnkEW666Saf2hwsOzub6urqkzb8bNGixWmvXVZW1ilfX1VVRXZ2NgkJCS7L684aci1/6rnnnqO4uJjrr7/eFRE9RkOuZVpaGo888ggLFy4kIMCn/0k8QUOu5a5du1i0aBFNmjRh+vTpZGdnc++993LkyBGfHjfTkGs5ePBgpk2bxg033EBZWRlVVVX8/Oc/51//+ldjRPY4XtkyM2nSJBwOxxlvK1euBMDhcJz0fsuyTvn4T1VWVnLjjTdSU1PDlClTnP7f4Ql+ep3Odu1O9fpTPe6L6nsta73//vtMmjSJDz/88JTFuS8612tZXV3NTTfdxFNPPUVqampjxfMo9fm+rKmpweFwMG3aNC644AKuuOIKnn/+ed566y2fb52B+l3LzZs3c//99/PEE0+watUqZs6cSXp6uvYlPA2v/DNk/Pjx3HjjjWd8Tdu2bVm/fj0HDx486bnDhw+fVEH/VGVlJddffz3p6enMmTPHp1plAGJiYvD39z/pr4pDhw6d9trFx8ef8vUBAQE0b97cZVndXUOuZa0PP/yQX//613z88ceMGjXKlTE9Qn2vZWFhIStXrmTNmjWMHz8esH8hW5ZFQEAA3333HSNHjmyU7O6mId+XCQkJtGrViqioqLrHOnfujGVZ7Nu3j5SUFJdmdlcNuZaTJ0/mwgsv5MEHHwSgR48ehIWFMXToUP7yl7/4bEv26Xhly0xMTAydOnU6461JkyYMGjSI/Px8fvzxx7r3Ll++nPz8fAYPHnza89cWMmlpaXz//fc++Ys4KCiIvn37Mnv27BMenz179mmv3aBBg056/XfffUe/fv0IDAx0WVZ315BrCXaLzO233857772nfvSj6nstIyMj2bBhA2vXrq273X333XTs2JG1a9eeMJ7O1zTk+/LCCy/kwIEDFBUV1T22fft2/Pz8aN26tUvzurOGXMuSkhL8/E78Fe3v7w8ca9GW4xgaeOw2LrvsMqtHjx7W0qVLraVLl1rdu3c/aWp2x44drc8++8yyLMuqrKy0fv7zn1utW7e21q5da2VmZtbdysvLTfwnGFM71fD111+3Nm/ebE2YMMEKCwuzdu/ebVmWZT3yyCPWrbfeWvf62qnZv/vd76zNmzdbr7/+uqZmH1Xfa/nee+9ZAQEB1ssvv3zC92BeXp6p/wS3Ud9r+VOazXRMfa9lYWGh1bp1a+u6666zNm3aZM2fP99KSUmx7rjjDlP/CW6jvtfyzTfftAICAqwpU6ZYO3futBYtWmT169fPuuCCC0z9J7g1ny9mcnJyrJtvvtmKiIiwIiIirJtvvtnKzc094TWA9eabb1qWZVnp6ekWcMrb3LlzGz2/aS+//LKVlJRkBQUFWX369LHmz59f99y4ceOsiy666ITXz5s3z+rdu7cVFBRktW3b1nrllVcaObH7qs+1vOiii075PThu3LjGD+6G6vt9eTwVMyeq77XcsmWLNWrUKCskJMRq3bq1NXHiRKukpKSRU7un+l7Lf/7zn1aXLl2skJAQKyEhwbr55putffv2NXJqz+CwLLVXiYiIiOfyyjEzIiIi4jtUzIiIiIhHUzEjIiIiHk3FjIiIiHg0FTMiIiLi0VTMiIiIiEdTMSMiIiIeTcWMiIiIeDQVMyIiIuLRVMyIiIiIR1MxIyIiIh5NxYyIiIh4tP8H8GZDpwmP3gAAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.histplot(pos_cross_sims_s, color='green',kde=True)\n",
    "sns.histplot(neg_cross_sims_s.sample(N), color='red',kde=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a9b8002-1cb8-4899-a903-c43fabf37ddc",
   "metadata": {},
   "source": [
    "## 输出具体的结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "7f5224be-74af-410b-b165-e211399bc26b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def similarity(v1,v2):\n",
    "    dot_product = np.dot(v1, v2)\n",
    "\n",
    "    magnitude_v1 = np.linalg.norm(v1)\n",
    "    magnitude_v2 = np.linalg.norm(v2)\n",
    "\n",
    "    return dot_product / (magnitude_v1 * magnitude_v2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "833ee658-32b0-49ab-b6eb-2c9b258e1762",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question:口袋奇兵是什么？\n",
      "Answer:世界上最好玩的游戏，江娱互动旗下的一款SLG游戏\n",
      "0.5740935802459717\n",
      "\n",
      "Question:芭芭拉\n",
      "Answer:口袋奇兵5月份最新推出的空军英雄，既美丽又强大，触发技能后最擅长连击击毙敌军。\n",
      "0.6880062818527222\n",
      "\n",
      "Question:阵容推荐、推荐个阵容\n",
      "Answer:\n",
      "1、空军：推荐芭芭拉、尤里卡、邦尼\n",
      "2、陆军：推荐尤里卡、流浪者、波里克\n",
      "3、海军：推荐蕾拉、蔻缇雅、暴风赤红\n",
      "0.6594719290733337\n",
      "\n",
      "Question:小游戏在哪\n",
      "Answer:目前只有10-20级玩家可以看到小游戏中心，您可以点击右上角的（大喇叭）常规活动，翻到最后面就可以看到了哦！\n",
      "0.5766243934631348\n",
      "\n",
      "Question:头像同步不了\n",
      "Answer:建议您尝试登陆游戏后，点击头像、右上角的三个点、选择设置、用户信息，然后切换一下用户信息，看看是否可以解决您的不能同步的问题。然后回到游戏后在游戏中点击左上角头像、再次点击头像、将头像栏拉动到最下方看看是否有您需要更换的头像。\n",
      "0.6618648171424866\n",
      "\n",
      "Question:误操作了怎么办\n",
      "Answer:军级达到5级、并且操作在7天之内(英雄类需操作两小时内并且英雄没有出征过)，90天内没有过申请撤销操作的请求才可以申请一次撤销操作哦~如果您符合条件，可以点击头像-联系客服，将您的情况详细描述一下\n",
      "0.5097072124481201\n",
      "\n",
      "Question:礼包码\n",
      "Answer:几个长期有效的：KDQB666、TOPWAR666、topwar888、KD2023  \n",
      "[温馨提示]点击头像→设置→礼品码，输入礼品码领取，即可直接进入您的背包中；\n",
      "0.6091381311416626\n",
      "\n",
      "Question:内部号、我想当托\n",
      "Answer:这个游戏良心运营，没有托儿\n",
      "0.5079904198646545\n",
      "\n",
      "Question:芭芭拉\n",
      "Answer:口袋奇兵5月份最新推出的空军英雄，既美丽又强大，触发技能后最擅长连击击毙敌军。\n",
      "0.6880062818527222\n",
      "\n",
      "Question:口袋奇兵是什么？\n",
      "Answer:世界上最好玩的游戏，江娱互动旗下的一款SLG游戏\n",
      "0.5740935802459717\n",
      "\n",
      "Question:芭芭拉\n",
      "Answer:口袋奇兵5月份最新推出的空军英雄，既美丽又强大，触发技能后最擅长连击击毙敌军。\n",
      "0.6880062818527222\n",
      "\n",
      "Question:口袋奇兵是什么？\n",
      "Answer:世界上最好玩的游戏，江娱互动旗下的一款SLG游戏\n",
      "0.5740935802459717\n",
      "\n",
      "Question:芭芭拉\n",
      "Answer:口袋奇兵5月份最新推出的空军英雄，既美丽又强大，触发技能后最擅长连击击毙敌军。\n",
      "0.6880062818527222\n",
      "\n",
      "Question:无法登陆\n",
      "Answer:①点击小程序右上角“···”(三个点的图标)→选择“重新进入小程序”；\n",
      "②切换4G/5G/WIFI进行尝试，或错开网络拥堵时段(如0点整)，稍候尝试；\n",
      "③尝试重启登录设备（手机、平板等）；\n",
      "④在小程序选择界面中，长按游戏图标，将小程序拖动删除，再重新搜索“口袋奇兵”。\n",
      "0.5282047986984253\n",
      "\n",
      "Question:概率低、运气差\n",
      "Answer:我们游戏中所有活动，都是有概率可以获取奖励预览中的所有展示奖励的哦。\n",
      "相关概率，所有玩家都是一样的呢~\n",
      "您可以多尝试，一定有机会中大奖的哦！(～￣▽￣)～\n",
      "0.5829479694366455\n",
      "\n",
      "Question:战锤集结\n",
      "Answer:点击右侧的联盟、战争、自动加入、就可以看到加入自动上车功能了\n",
      "0.5071412324905396\n",
      "\n",
      "Question:换区\n",
      "Answer:一般是开服120天（121天起）之后开启迁服功能。需要您的人物等级达到80级才能转区~\n",
      "开启迁出功能的服务器可以在世界地图（能看到各个服务器的那个地图）查询哦。\n",
      "请您点进 世界，依次点击：左上角的“小地图”右边的小图标→右下角第二个图标 圆圈圈→左下角第二个按钮，查看迁服列表。\n",
      "【后方阵线】一栏里的战区就是您可以转区的哦~选择您想要转入的战区，如果满足调动要求，点击【准备调动】之后会显示相关说明和所需“调动申请书”的数量\n",
      "0.5703070163726807\n",
      "\n",
      "Question:怎么改名称\n",
      "Answer:Answer:请您点击左上角的 头像（个人信息），右上角有个小小的方框图标，点击就可以修改名字啦~\n",
      "头像-性别-右边的小方框，就可以修改性别啦~\n",
      "0.6675779819488525\n",
      "\n",
      "Question:专属技能碎片在哪里获得？\n",
      "Answer:专属技能碎片可以通过多种途径获得，例如礼包商城-特惠礼包界面可以购买专属技能碎片礼包\n",
      "0.8391760587692261\n",
      "\n",
      "Question:不能快速删除工厂（比如兵工厂、船厂、机场），要一个一个点太麻烦了\n",
      "Answer:我们非常感谢您能与我们分享您的想法。口袋奇兵团队全员将会认真听取您的意见，努力改进与优化，为打造更加公平多彩的游戏环境而努力。\n",
      "期待您不吝赐教，如果您还有其他好的意见与建议，欢迎随时告诉我们。\n",
      "小芙会将您的提议搜集后上报给开发团队，让开发团队在后续的优化或者更新中优先考虑您的提议加以改进，再次感谢您的反馈！\n",
      "0.43950924277305603\n",
      "\n",
      "Question:为什么我删除不了角色？\n",
      "Answer:在同一绑定账号下，当前账号必须保留一个游戏角色才可以删除另外一个角色，\n",
      "您可以登陆游戏点击头像、设置、角色管理、就可以选择创建角色或删除角色，如果没有删除角色按键，\n",
      "可能是因为您并不是在当前app创建的角色，您需要回到创建账号的小程序或app等才可以删除角色哦~\n",
      "0.7761311531066895\n",
      "\n",
      "Question:怎么帮小号送给别人\n",
      "Answer:平台不支持转送或者账号交易\n",
      "0.47375765442848206\n",
      "\n",
      "Question:三个强化，伤害加深，背包只显示一个，在安装强化界面能看到3个，但是不能合成\n",
      "Answer:您这边需要把强化部件方案1,2,3,4需要合成的部件全部取下来然后再合成\n",
      "0.6903927326202393\n",
      "\n",
      "Question:在哪里能看到我刚刚获得的装备图纸\t\n",
      "Answer:您可以在军火库制作对应装备的界面查看到您拥有的图纸\n",
      "0.658401608467102\n",
      "\n",
      "Question:我购买的资源不见了\t\n",
      "Answer:礼包购买或者箱子开出的资源道具会出现在背包里，其他资源会直接加入到资源总数中喔\n",
      "0.6858749389648438\n",
      "\n",
      "Question:为什么我买了精英月卡却没有获得钻石\t\n",
      "Answer:精英月卡中并不包含钻石喔，精英月卡的特权是不用看视频，直接获得当天看视频可以获得的钻石，如果已经通过看视频获得了钻石，购买精英月卡时就不会获得这部分钻石了噢\n",
      "0.7498976588249207\n",
      "\n",
      "Question:你们是真的吃相难看，就这样对你们的衣食父母？\t\n",
      "Answer:指挥官大人您好！我们的活动是有一定概率的哦，非常抱歉未能完全满足您的期望。我们一直在努力，希望您能继续享受我们精彩的游戏体验！这边祝您后续欧气满满，好运常在\n",
      "0.5918810963630676\n",
      "\n",
      "Question:我使用了自选兵种+1/+2宝箱没有获得士兵\t\n",
      "Answer:使用道具并选择您想要的士兵后，请您到背包里‘士兵’的选项里查看噢\n",
      "0.617608368396759\n",
      "\n",
      "Question:为什么我的修理厂是空的，受伤的士兵却没有收进去\t\n",
      "Answer:如果是在攻击其他玩家基地时产生的伤兵，那不会进入修理厂喔，VIP13-VIP16的特权可以按一定比例转化成伤兵进入修理厂\n",
      "0.6001214385032654\n",
      "\n",
      "Question:为什么我用了两个额外队列的道具，只增加了一个队列\t\n",
      "Answer:如果使用多个额外限时队列道具，只会增加限时队列的持续时间，并不会增加队列数量。\n",
      "0.6737362146377563\n",
      "\n",
      "Question:礼品码怎么使用，在哪领取\t\n",
      "Answer:点击头像-设置-礼品码进行兑换使用，获取方式请留意后续公告牌与游戏内公告邮件\n",
      "0.6476421356201172\n",
      "\n",
      "Question:我的获得的雷神元素背包里怎么看不见\t\n",
      "Answer:需要80级之后解锁超级武器实验室建筑之后才可进行查看，80级前获得的雷神元素会正常累计增加\n",
      "0.6691535115242004\n",
      "\n",
      "Question:装饰属性在城市详情内怎么不显示啊\t\n",
      "Answer:装饰属性是在神奇宝箱-buff界面进行显示的\n",
      "0.786723792552948\n",
      "\n",
      "Question:旧的装饰套装比如假日酒店等能否返厂让人再次购买呢\t\n",
      "Answer:计划后续会增加获取途径，建议可以通过公告牌留意后续活动奖励以及礼包\n",
      "0.525277853012085\n",
      "\n",
      "Question:黄金坦克\n",
      "Answer:黄金坦克附体，坦克血量大增。生效数量最多：1\n",
      "0.7387195825576782\n",
      "\n",
      "Question:双子摇摇马的功能\n",
      "Answer:（有几率）破损坦克修理时不降级。生效数量最多：1\n",
      "0.6075373291969299\n",
      "\n",
      "Question:指挥官雕像\n",
      "Answer:（有几率）训练坦克时立即完成。生效数量最多：1\n",
      "0.5459924936294556\n",
      "\n",
      "Question:礼物箱\n",
      "Answer:训练速度增加。生效数量最多：5\n",
      "0.6708224415779114\n",
      "\n",
      "Question:长椅\n",
      "Answer:行军速度加快。生效数量最多：5\n",
      "0.8136347532272339\n",
      "\n",
      "Question:幸运泉\n",
      "Answer:金币产出增加。生效数量最多：5\n",
      "0.6920217275619507\n",
      "\n",
      "Question:高速公路\n",
      "Answer:对黑暗军团行军速度略微加快。生效数量最多：100\n",
      "0.6422338485717773\n",
      "\n",
      "Question:紫色地板\n",
      "Answer:训练速度略微增加。生效数量最多：100\n",
      "0.5185257196426392\n",
      "\n",
      "Question:灯笼\n",
      "Answer:金币产出略微增加。生效数量最多：10\n",
      "0.5136443972587585\n",
      "\n",
      "Question:青苹果树\n",
      "Answer:修理速度略微增加。生效数量最多：20\n",
      "0.5166247487068176\n",
      "\n",
      "Question:环太平洋联动活动后续还会有三期吗？\n",
      "Answer:您好，本次环太平洋联动活动为最后一次返场。为了避免不必要的损失，本次活动道具还请在活动结束前及时兑换使用。\n",
      "0.6672731041908264\n",
      "\n",
      "Question:我怎么找不到异界能源研究所了？\n",
      "Answer:您好，异界能源研究所已合并至机甲猎人研究所页面。您可以前往机甲猎人研究所中，在页面右上方位置找到异界能量研究所图标并点击进入。\n",
      "0.7120376229286194\n",
      "\n",
      "Question:为啥别的玩家能看到游戏里主动推送的联动活动相关视频，我号上怎么看不到？\n",
      "Answer:您好，感谢您的反馈。每位玩家可以点开联动活动页面左上角的作战影像观看活动相关完整视频。\n",
      "0.617574155330658\n",
      "\n",
      "Question:红色能源碎片如何获得？\n",
      "Answer:您好，精炼能源碎片可用于在物资补给站中获得兑换高级奖励。研究红色能量试管必得精炼能源碎片或者击败遗迹内怪兽有概率获得精炼能源碎片。\n",
      "0.7186340689659119\n",
      "\n",
      "Question:我在研究物资补给站误兑换了东西，可以帮我申请误操作处理吗？\n",
      "Answer:您好，可以按照三个月一次的误操作机会为您扣除所得道具并返还所使用的对应能源碎片，但是请注意每样道具的兑换次数无法重置。\n",
      "0.6458031535148621\n",
      "\n",
      "Question:我如何往遗迹深渊中进驻多个队列？\n",
      "Answer:您好，每位玩家在遗迹深渊中只能进驻一个行军队列及相应部队。进驻到遗迹深渊的部队和行军队列会被实际占用，无法在原地图其他玩法中使用。\n",
      "0.6925995945930481\n",
      "\n",
      "Question:给5个机甲猎人充能有什么作用？\n",
      "Answer:您好，毒妇血量较高，玩家可配合机甲猎人进行击杀，并可根据机甲猎人分别的能量值剩余情况选择给任意机甲猎人充能。每日前10个异界能源的提交会获得一份奖励。\n",
      "5个机甲猎人捐献共享每日可获得奖励的次数。即捐献任意一个机甲猎人拿到每日全部次数奖励后，捐献其他机甲也无法再获得次数奖励。\n",
      "0.5138034224510193\n",
      "\n",
      "Question:[坚定]效果，[专注]效果和[加固]效果分别是什么？\n",
      "Answer:[坚定]效果：无论是否出战，增加全部单位生命；[专注]效果：无论是否出战，增加全部单位攻击；[加固]效果：无论是否出战，增加全部单位伤害减免。\n",
      "拥有多个[坚定]、[专注]、[加固]效果时，只会生效一个[坚定]、[专注]、[加固]效果，其中加成最高的[坚定]、[专注]、[加固]效果生效。\n",
      "0.863585889339447\n",
      "\n",
      "Question:如何增加切尔诺阿尔法的[坚定]，[专注]和[加固]效果？\n",
      "Answer:英雄升星会增加[坚定]效果，英雄5星时[坚定]效果提升至30%。\n",
      "英雄专属技能拥有[专注]和[加固]效果，英雄专属技能1级可获得[专注]效果1%，英雄专属技能10级时[专注]效果可提升至90%。加固效果是专属附加属性，专属技能5级时英雄可获得加固效果4%，专属技能7级时英雄[加固]效果在4%的基础上增加12%，共计16%。\n",
      "0.7647935748100281\n",
      "\n",
      "Question:什么是[援护]效果？\n",
      "Answer:拥有[援护]效果的单位存活时，同列其它单位受到的伤害减少，减少的伤害由[援护]单位承担（尤里卡突袭者还可将分担伤害减少40%），释放顺序低于洛克菲尔德技能。\n",
      "0.6907435059547424\n",
      "\n",
      "Question:尤里卡突袭者的[援护]效果是否能分担各种英雄技能打出的伤害？\n",
      "Answer:您好，[援护]效果不分担燃烧伤害（除燃烧伤害外，都可以分担）。拥有点燃技能的英雄目前有寂，914，马克西莫，异化娜迪亚等。\n",
      "0.5859891176223755\n",
      "\n",
      "Question:升星或升级是否可以增加尤里卡突袭者的[援护]效果？\n",
      "Answer:您好，升星或升级无法增加该英雄的[援护]效果。只有配置尤里卡突袭者7级专属技能时可以将该英雄的分担伤害减少效果从40%提升至60%。\n",
      "0.817891538143158\n",
      "\n",
      "Question:探戈狼技能礼包的价格档位有哪些，各包含多少个专属技能？\n",
      "Answer:您好，该礼包有98元和128元两个价格档位，分别包含300个和450个探戈狼1级专属技能。\n",
      "0.7253335118293762\n",
      "\n",
      "Question:探戈狼技能后续是否可以在专属技能商店中获得？\n",
      "Answer:您好，本次联动活动期间玩家只可以通过礼包购买的时候获得探戈狼专属技能。预计后续可以在专属技能商店中兑换获得该英雄技能。\n",
      "0.8022236824035645\n",
      "\n",
      "Question:获得探戈狼以及它的专属技能后，是否会影响新迁服功能上线后我迁服所需的调动申请书数量？\n",
      "Answer:您好，获得探戈狼以及它的专属技能不会影响后续迁服时所需调动申请书的数量，还请您知悉。\n",
      "0.8008280396461487\n",
      "\n",
      "Question:为何我点击活动时提示受政策原因，该地区IP无法参与活动？\n",
      "Answer:您好，官方坚决遵守各国家及地区对内容管理的政策要求，若部分内容无法正常体验，请您更换IP并重新登录尝试，具体方法可与其他玩家或社群进行交流。\n",
      "0.5046846866607666\n",
      "\n",
      "Question:什么是秘密武器？为什么我们服没有秘密武器？\n",
      "Answer:为了给新手玩家更多的帮助，辅助成长，目前部分新服会开放“秘密武器”功能，可以提供士兵生产、护盾开启、空投宝箱等特技，帮助您快速发展。\n",
      "目前秘密武器功能还在发展初期，仅在部分服务器开放，感谢您的理解。我们也会请团队考虑后续为更多服务器开启此功能，敬请期待。\n",
      "0.7733469605445862\n",
      "\n",
      "Question:秘密武器的开放条件是什么？\n",
      "Answer:需要指挥官基地等级到达12级，通过在基地内开启对应地块即可获得该建筑。\n",
      "0.5704920887947083\n",
      "\n",
      "Question:秘密武器如何组装？基础部件道具如何获得？\n",
      "Answer:您可以点击秘密武器建筑，在第一个页签“机械组装”中查看所需基础部件道具，包括无后坐力炮、多管火箭弹、数字阵列雷达、核心动力泵、多地形地盘、重甲控制舱。\n",
      "在第二个页签“秘密部署”中完成对应的任务（包括雷达任务、训练士兵、合成建筑等）后，即可领取对应的基础部件，回到第一页签进行组装。\n",
      "0.8004412055015564\n",
      "\n",
      "Question:秘密武器的三个特技介绍。\n",
      "Answer:智能训练：点击使用智能训练特技，启动后会有2分钟的时间，在这个时间内点击添加新的造兵任务都会即刻生产完毕。不过需要注意是每次使用均有冷却时间以及快速造兵的数量上限。\n",
      "援军定位：使用特技后,可立即进行一次空投支援,获得随机奖励。空投次数每480分钟回复一次,最多拥有3次。点击技能旁边的感叹号可了解空投获取的随机奖励内容。\n",
      "中枢壁垒：使用秘密武器的能源开启60分钟护盾, 开启后,敌人无法对您的基地发起进攻。冷却时间:2880分钟。\n",
      "0.5943860411643982\n",
      "\n",
      "Question:秘密武器特技的冷却时间、上限之类的数值，是固定的吗？会随着升级变化吗？\n",
      "Answer:您好，这些数值是固定的。\n",
      "0.5146230459213257\n",
      "\n",
      "Question:秘密武器在危险地带提示不能开启护盾，是正常的吗？\n",
      "Answer:您好，秘密武器的护盾功能与游戏本身限制相同。游戏内如危险地带等本身无法开启护盾的地点，秘密武器的护盾也无法开启。是游戏的正常设定。\n",
      "0.7784391641616821\n",
      "\n",
      "Question:秘密武器如何升级？\n",
      "Answer:您可以通过完成秘密部署任务和开启地块，获得稀有金属零件来进行升级。任务奖励中还包括钻石、核心设计图等不错的奖励，十分建议您优先参与并完成。\n",
      "0.594862163066864\n",
      "\n",
      "Question:秘密武器最高等级是多少？\n",
      "Answer:您好，目前秘密武器最高可升到50级。\n",
      "0.7704099416732788\n",
      "\n",
      "Question:稀有金属零件怎么获得？\n",
      "Answer:稀有金属零件可以通过完成秘密部署任务，或者开启基地内地块获得。（注：基地内第二岛屿仅有20个左右的部分地块解锁时会提供稀有金属零件。）\n",
      "0.7430362701416016\n",
      "\n",
      "Question:秘密部署的任务，我只领取上面的总奖励了，下面的小任务的奖励我忘记领了。\n",
      "Answer:您好，当您领取秘密部署总任务的奖励时，下方未领取的子任务奖励会一并帮您领取的。请不要担心，您前面完成的秘密部署任务的奖励已经全部领取到了。\n",
      "0.7813732624053955\n",
      "\n",
      "Question:秘密部署中的帝国宝藏任务无法完成！\n",
      "Answer:非常抱歉指挥官，曙光之地里搜不到帝国宝藏。您可能需要等曙光之地结束后回到原服才能继续攻打帝国宝藏。我们也会将此问题反馈给团队，请团队考虑后续优化此任务。感谢您的理解与支持。\n",
      "0.6322575807571411\n",
      "\n",
      "Question:秘密武器合成等级和建造科技等级一致的情况下，无法完成“合成≥N级建筑”任务怎么办？\n",
      "Answer:您好，建议您将“合成”科技的等级优先提升一级，在基地建造多个低等级兵厂建筑后，拖动建筑合成以完成该任务。  前期资源有限，建议您优先升级“合成等级”科技，从而更快的成长。\n",
      "0.6219929456710815\n",
      "\n",
      "Question:重装机兵模块研究活动会持续多久？\n",
      "Answer:您好，破浪突袭者BW-3的基础机体、复合模块、战斗模块、传动模块需要通过重装机兵模块研究活动获得，在活动页面消耗研究次数以增加模块研究进度，有几率暴击。当研究进度达到100%时，即可获得当前研究的模块，多余的进度会自动累计到下一模块的研究进度中。\n",
      "若指挥官未全部获取4个部件，该活动页面不会出现结束倒计时（会一直存在），直至指挥官全部获得4个模块后，该活动页面才会出现结束倒计时（48h）。\n",
      "0.6693931818008423\n",
      "\n",
      "Question:如何获得重装机兵模块研究次数？\n",
      "Answer:您好，指挥官可通过【完成每日任务】和【购买模块研究礼包】2种方式来获得重装机兵模块研究次数。\n",
      "0.8398867845535278\n",
      "\n",
      "Question:研究进程结束后，多余的研究次数怎么办？\n",
      "Answer:您好，研究进程结束后，多余研究次数将自动转化为钻石，指挥官可前往【超值活动-活动领奖中心】领取。\n",
      "0.6442395448684692\n",
      "\n",
      "Question:破浪突袭者BW-3的4个基础模块需要多久才能都获得嘛？其它的模块怎么获得？\n",
      "Answer:您好，模块研究进度会受暴击因素影响而浮动，因此每位玩家所需的获取时间可能会有不同。指挥官也可以选择购买模块研究礼包的方式来加速获得4个基础模块。至于其它的模块，通过【荒野行动】会概率掉落，以及开启【艾斯道刻穿甲剑重装芯片宝箱】可以有概率获取。\n",
      "0.6554750204086304\n",
      "\n",
      "Question:小蓝是拥有控制技能的重装机兵，这个控制技能怎么理解？它更适合攻击还是防守？\n",
      "Answer:您好，破浪突袭者BW-3的控制技能主要体现在 技能中有一项是潮汐震荡，如果被击目标处在攻击力降低状态，50%概率对其附加[眩晕]（无法普攻和释放主动技能）。从技能定位来看，其更偏向攻击，但防守效果也不错。\n",
      "0.6177594065666199\n",
      "\n",
      "Question:你们这个新机甲的获取时间周期也太长了吧?不花钱就不能快速获得，你们也太过分了吧\n",
      "Answer:您好！非常感谢您的反馈。我们非常理解您所描述的问题。本次重装机兵模块获取方式确实不同以往，一成不变的游戏模式可能会让指挥官感到疲劳，因此我们一直在寻找新的游戏模式，希望为玩家带来更多的乐趣。我们会认真倾听玩家的反馈，并始终努力改进我们的游戏以符合您的期望。感谢您的信任与支持。\n",
      "0.4816896915435791\n",
      "\n",
      "Question:如何获得艾斯道刻穿甲剑芯片？\n",
      "Answer:您好，5月25日零点后重装芯片宝箱、重装秘藏、芯片超频有几率开出艾斯道刻穿甲剑重装芯片。开启艾斯道刻穿甲剑重装芯片宝箱（参与5月25日零点上线的劲爆囤货节活动获得），或者购买相关礼包也可以获得艾斯道刻穿甲剑芯片。\n",
      "0.7271589040756226\n",
      "\n",
      "Question:征服者活动的匹配机制是什么？\n",
      "Answer:在征服者活动里，地图是重新分配的，参与服务器的地图会【随机】打乱成一个新的地图。征服者系统是由多个轮次组成的连续战役活动，宣战后可确定本轮次的征服目标。只可以对本势力战区相邻的其他势力战区进行宣战。\n",
      "0.6017512679100037\n",
      "\n",
      "Question:什么是势力？势力霸主是什么，有什么作用？\n",
      "Answer:势力是战区的集合。在活动一开始时，每个战区各为一个独立势力，随着战事不断推进，战区之间相互吞并，会出现由许多战区构成的势力，吞并其他战区的战区称为宗主战区，被吞并的战区叫做附属战区。势力能够包含的战区数量无上限。势力霸主就是一个势力中宗主战区的首领，拥有绝对的领导权，可以任命官职，组建军团并任命军团长。\n",
      "0.8214671611785889\n",
      "\n",
      "Question:对某个战区宣战后是否可以取消宣战？\n",
      "Answer:势力的管理层可以在活动页面进行宣战，宣战时会弹出二次确认面板，宣战成功后不可以取消宣战。\n",
      "0.6400095820426941\n",
      "\n",
      "Question:势力排名以什么为标准？如何增加势力声望？\n",
      "Answer:每一轮战役结束后，会以势力排名为基准，对势力发奖。势力排名以势力国家数量为排名基准，在势力版图中包含的战区数量相同情况下，势力声望将决定势力排名顺序。积极参与战备任务和势力兵营会增加势力声望。\n",
      "0.8081082701683044\n",
      "\n",
      "Question:军团是什么，有什么用，怎么组建军团？ \n",
      "Answer:以个人为单位申请，细化势力部队，方便战斗指挥，一个势力的军团数量上限与所拥有的附属战区数量有关。势力霸主可以任命军团长用来设置军团成员，方便军团作战的指挥。\n",
      "0.6972187161445618\n",
      "\n",
      "Question:当一个势力把另一个势力的宗主战区吞并，它能吞并该势力下的所有战区吗？\n",
      "Answer:不能的，当一个宗主战区被吞并时，该势力下的所有附属战区变会为独立战区。\n",
      "0.7655807137489319\n",
      "\n",
      "Question:贡献积分和军功怎么获得，可以用来做什么?\n",
      "Answer:指挥官完成每日任务和领取捐兵积分进度奖励可以获得贡献积分；在活动战场与敌方战区指挥官发生战斗，击杀敌方士兵和损失己方士兵都可以获得军功。两者都可以在征服者商店兑换奖励。\n",
      "0.6894623041152954\n",
      "\n",
      "Question:获得军功的数量与哪些因素有关？\n",
      "Answer:您好，主要与两个因素有关。\n",
      "1. 战斗场景：当玩家在战场中【杀死敌人士兵】时会获得军功，在不同场景下发生的战斗获得的军功数量不同，三个建筑内（首府+2个王者遗迹-守卫）和在战场区域内发生战斗的军功较多，在战场区域外发生战斗的军功较少。\n",
      "2. 士兵等级&基地等级：40级以下士兵没有军功，高于对方指挥官20级以后击杀敌方士兵不得军功，指挥官受伤的士兵等级低于指挥官自身5级后不会获得军功。\n",
      "0.6687365174293518\n",
      "\n",
      "Question:什么叫起义？什么时候可以起义呢？起义需要具备什么条件吗？\n",
      "Answer:本战区向宗主势力发起挑战，胜利后可脱离该势力并恢复成单独势力。开启起义准备后，有起义意愿的战区需要先进行【检阅】，满足条件才可进行起义。检阅不会消耗士兵，点击检阅按键后，自己基地里拥有并且满足条件的士兵，会计入到目标兵数里；只有在宣战阶段截止前审核通过的服务器才能宣战成功。起义宣告成功后会在战场公示阶段变成一个独立势力，宗主势力会自动对起义战区宣战。起义失败后仍然是附属战区。\n",
      "0.6850916147232056\n",
      "\n",
      "Question:最多可以同时和多少个战区宣战呢？有什么限制要求吗？\n",
      "Answer:势力的管理层可对一个或多个可以相邻战区（世界大地图可以通过直线相连）进行宣战标记。宣战阶段结束，这些标记的战区会被锁定。如果未进行任何宣战，系统会随机锁定一个战区作为宣战目标。\n",
      "0.6037830114364624\n",
      "\n",
      "Question:怎么判断防守失败和战斗胜者？\n",
      "Answer:当战区首府和至少1个王者遗迹--守卫被摧毁后，则该战区防守失败，成为附属战区。多个势力攻打某一个势力的情况下，假如该势力防守失败，在战斗进入结算状态后，会根据本次战斗进攻势力的拆除的城防值排名来决定本次战斗的胜者，出现同分的情况时，则依据势力总军功值决定胜者。\n",
      "在前线奋斗的时候，也要注意守住自己的战区，己方战区如果失守，即便打下别的战区也不算数。\n",
      "0.7212809324264526\n",
      "\n",
      "Question:先锋对决怎么玩，胜负规则是什么？\n",
      "Answer:玩家拥有建筑的占领权后，拥有【先锋】权限的玩家，可以开启对决。开启对决后，该玩家在此建筑中的所有部队都将会进入对决状态中，其他势力的拥有【先锋】权限的玩家，可以发起对决；当击败玩家所有进入对决状态的部队后，视为攻击成功。对决连续守住一定的攻击次数，或守住规定的时间，则对决成功，否则对决失败。对决获胜，可获得一定时间的无敌时间，并立即摧毁一定的城防值；对决失败，则击败对决的玩家获得建筑的控制权。\n",
      "0.7062654495239258\n",
      "\n",
      "Question:征服者活动的个人参与限制条件是什么？一个服可以无限进人吗？\n",
      "Answer:需要个人基地等级≥60级后才能进行参与该活动。一个服不可以无限进人，会有人数限制。\n",
      "0.6693925857543945\n",
      "\n",
      "Question:我跨服进入战场后，我的坐标会在哪个位置？\n",
      "Answer:战斗阶段开启后，玩家可以通过活动界面或跨服按键进入目标战场，进入目标战场后会随机分配坐标；在战斗阶段期间，非原服玩家无法开盾；在战场区域不能开盾。\n",
      "0.5842746496200562\n",
      "\n",
      "Question:战斗阶段是否可以攻击其他战区指挥官？\n",
      "Answer:全力交锋--战斗阶段开始后，跨服到其它战区后，除了攻打首府和两个2个王者遗迹-守卫外（其他遗迹和发射塔为和平状态，不可攻击），还可以攻打该服玩家的城堡，采集田和强化弱化塔。\n",
      "0.5541176199913025\n",
      "\n",
      "Question:表彰奖励和封赏奖励有什么区别？\n",
      "Answer:每轮全力交锋--结算阶段后，根据势力排名宗主战区将获得用于表彰的奖励，由势力首领分发表彰给本战区指挥官。\n",
      "当势力首领封赏奖励给附属战区后，附属战区获得本战区表彰奖励，由附属战区首领分发给本战区指挥官。\n",
      "0.6605383157730103\n",
      "\n",
      "Question:如何开启装饰物商店功能？\n",
      "Answer:您好，所在服开服时间≥84天或已经开过一次曙光之地活动后才可以在金融中心里看到装饰物商店页签，且需要满足玩家基地等级≥60级后才可以解锁装饰物商店功能。\n",
      "0.6786451935768127\n",
      "\n",
      "Question:为什么我们服没有全息幻视功能？\n",
      "Answer:您好，需要玩家所在服开完一次曙光之地活动后才能使用该功能\n",
      "0.6019400954246521\n",
      "\n",
      "Question:橙色装备改造功能的开启条件是什么？\n",
      "Answer:需要玩家装备4件橙色装备后，才能开启橙色装备改造功能。符合开启条件后玩家可以在军火库改造页面内将装备进行进阶操作。装备改造后可以获得额外属性，额外属性在佩戴装备后可对对应兵种生效。\n",
      "0.742383599281311\n",
      "\n",
      "Question:每个橙色装备最多可以改造到多少阶？\n",
      "Answer:目前的设定是每个橙色装备最多可以改造到8阶。4件橙色装备改造品阶总和达到一定值后可以激活额外套装属性效果\n",
      "0.8099770545959473\n",
      "\n",
      "Question:分解已经改造过的橙色装备时，改造材料会返还吗？\n",
      "Answer:分解已经改造过的橙色装备时，装备制造和改造消耗的材料、以及装备改造消耗的改造图纸会返还，但是装备制造图纸不会返还~\n",
      "0.9126039147377014\n",
      "\n",
      "Question:装备改造是什么？装备怎么改造\n",
      "Answer:可以在军火库改造页面内将装备进行进阶操作，装备改造后可以获得额外属性，额外属性在佩戴装备后可对对应兵种生效。\n",
      "0.5781945586204529\n",
      "\n",
      "Question:新区开放\n",
      "Answer:我们会不定期开放新服务器，您可以留意下头像-设置-角色管理里的服务器列表\n",
      "0.44808635115623474\n",
      "\n",
      "Question:为什么我买了至尊永久卡后，永久卡头像框不能设置\t\n",
      "Answer:需要您在背包里使用该道具，再到个人中心页面或者聊天频道里进行设置\n",
      "0.5235860347747803\n",
      "\n",
      "Question:商店购买的石油粮食为什么在背包里找不到呢\t\n",
      "Answer:从商店里购买的石油粮食不会在背包内显示哦，会直接加到总量里的\n",
      "0.7838613986968994\n",
      "\n",
      "Question:我经常忘记切换强化，有什么建议或者改善方法吗\t\n",
      "Answer:感谢您的反馈，我们会把您的想法转达给团队的，后期可能会添加一个出征时强化选择提醒功能\n",
      "0.5529630184173584\n",
      "\n",
      "Question:在哪里能看到我刚刚获得的装备图纸\t\n",
      "Answer:您可以在军火库制作对应装备的界面查看到您拥有的图纸\n",
      "0.6584015488624573\n",
      "\n",
      "Question:我购买的资源不见了\t\n",
      "Answer:礼包购买或者箱子开出的资源道具会出现在背包里，其他资源会直接加入到资源总数中喔\n",
      "0.6858749389648438\n",
      "\n",
      "Question:为什么我买了精英月卡却没有获得钻石\t\n",
      "Answer:精英月卡中并不包含钻石喔，精英月卡的特权是不用看视频，直接获得当天看视频可以获得的钻石，如果已经通过看视频获得了钻石，购买精英月卡时就不会获得这部分钻石了噢\n",
      "0.7498976588249207\n",
      "\n",
      "Question:自选兵种+1/+2宝箱获得的士兵等级是怎么算的？\t\n",
      "Answer:是根据您的个人科技【士兵单位】的等级计算+1/+2，比如您当前的个人科技中【陆军单位】等级是80，可以通过该道具获得81/82级陆军士兵。\n",
      "0.7085302472114563\n",
      "\n",
      "Question:我使用了自选兵种+1/+2宝箱没有获得士兵\t\n",
      "Answer:使用道具并选择您想要的士兵后，请您到背包里‘士兵’的选项里查看噢\n",
      "0.617608368396759\n",
      "\n",
      "Question:为什么我的修理厂是空的，受伤的士兵却没有收进去\t\n",
      "Answer:如果是在攻击其他玩家基地时产生的伤兵，那不会进入修理厂喔，VIP13-VIP16的特权可以按一定比例转化成伤兵进入修理厂\n",
      "0.6001214385032654\n",
      "\n",
      "Question:为什么我用了两个额外队列的道具，只增加了一个队列\t\n",
      "Answer:如果使用多个额外限时队列道具，只会增加限时队列的持续时间，并不会增加队列数量。\n",
      "0.6737362146377563\n",
      "\n",
      "Question:礼品码怎么使用，在哪领取\t\n",
      "Answer:点击头像-设置-礼品码进行兑换使用，获取方式请留意后续公告牌与游戏内公告邮件\n",
      "0.6476421356201172\n",
      "\n",
      "Question:我的获得的雷神元素背包里怎么看不见\t\n",
      "Answer:需要80级之后解锁超级武器实验室建筑之后才可进行查看，80级前获得的雷神元素会正常累计增加\n",
      "0.6691535115242004\n",
      "\n",
      "Question:装饰属性在城市详情内怎么不显示啊\t\n",
      "Answer:装饰属性是在神奇宝箱-buff界面进行显示的\n",
      "0.786723792552948\n",
      "\n",
      "Question:旧的装饰套装比如假日酒店等能否返厂让人再次购买呢\t\n",
      "Answer:计划后续会增加获取途径，建议可以通过公告牌留意后续活动奖励以及礼包\n",
      "0.525277853012085\n",
      "\n",
      "Question:黄金坦克\n",
      "Answer:黄金坦克附体，坦克血量大增。生效数量最多：1\n",
      "0.7387195825576782\n",
      "\n",
      "Question:双子摇摇马的功能\n",
      "Answer:（有几率）破损坦克修理时不降级。生效数量最多：1\n",
      "0.6075373291969299\n",
      "\n",
      "Question:指挥官雕像\n",
      "Answer:（有几率）训练坦克时立即完成。生效数量最多：1\n",
      "0.5459924936294556\n",
      "\n",
      "Question:礼物箱\n",
      "Answer:训练速度增加。生效数量最多：5\n",
      "0.6708224415779114\n",
      "\n",
      "Question:长椅\n",
      "Answer:行军速度加快。生效数量最多：5\n",
      "0.8136347532272339\n",
      "\n",
      "Question:幸运泉\n",
      "Answer:金币产出增加。生效数量最多：5\n",
      "0.6920217275619507\n",
      "\n",
      "Question:高速公路\n",
      "Answer:对黑暗军团行军速度略微加快。生效数量最多：100\n",
      "0.6422338485717773\n",
      "\n",
      "Question:紫色地板\n",
      "Answer:训练速度略微增加。生效数量最多：100\n",
      "0.5185257196426392\n",
      "\n",
      "Question:灯笼\n",
      "Answer:金币产出略微增加。生效数量最多：10\n",
      "0.5136443972587585\n",
      "\n",
      "Question:青苹果树\n",
      "Answer:修理速度略微增加。生效数量最多：20\n",
      "0.5166247487068176\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(input_question)):\n",
    "    sims = similarity(emb_answer[i],emb_question[i])\n",
    "    print(f\"Question:{input_question[i]}\\nAnswer:{input_answer[i]}\\n{sims}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd63deec-e324-470e-aad4-ff13cabcbaba",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "348b3ab6-f42d-4c54-a9b7-3a6223387168",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[{'corpus_id': 93, 'score': 0.4407898187637329},\n",
       "  {'corpus_id': 63, 'score': 0.2768343389034271},\n",
       "  {'corpus_id': 76, 'score': 0.27134737372398376}]]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "input_queries = ['怎么攻击别人？']\n",
    "q_embedding = modelB.encode(input_queries)\n",
    "results = util.semantic_search(query_embeddings = q_embedding,corpus_embeddings= emb_answer,top_k=3)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "37fbf424-9b29-44ba-8300-65aab4b56a35",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4407898"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity(q_embedding[0],emb_answer[93])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "id": "6b7015ab-0ab1-4b5a-bb70-1f1eff19c8cf",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('全力交锋--战斗阶段开始后，跨服到其它战区后，除了攻打首府和两个2个王者遗迹-守卫外（其他遗迹和发射塔为和平状态，不可攻击），还可以攻打该服玩家的城堡，采集田和强化弱化塔。', '战斗阶段是否可以攻击其他战区指挥官？') score:0.4162951409816742\n",
      "('玩家拥有建筑的占领权后，拥有【先锋】权限的玩家，可以开启对决。开启对决后，该玩家在此建筑中的所有部队都将会进入对决状态中，其他势力的拥有【先锋】权限的玩家，可以发起对决；当击败玩家所有进入对决状态的部队后，视为攻击成功。对决连续守住一定的攻击次数，或守住规定的时间，则对决成功，否则对决失败。对决获胜，可获得一定时间的无敌时间，并立即摧毁一定的城防值；对决失败，则击败对决的玩家获得建筑的控制权。', '先锋对决怎么玩，胜负规则是什么？') score:0.33781808614730835\n",
      "('您好，破浪突袭者BW-3的控制技能主要体现在 技能中有一项是潮汐震荡，如果被击目标处在攻击力降低状态，50%概率对其附加[眩晕]（无法普攻和释放主动技能）。从技能定位来看，其更偏向攻击，但防守效果也不错。', '小蓝是拥有控制技能的重装机兵，这个控制技能怎么理解？它更适合攻击还是防守？') score:0.29779693484306335\n"
     ]
    }
   ],
   "source": [
    "for ret in results[0]:\n",
    "    print(f\"{all_datas[ret['corpus_id']]} score:{ret['score']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec80db02-8714-4132-8457-2acf024b1a59",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2d47bd28-2482-47cb-b058-9f2f8bce9944",
   "metadata": {},
   "source": [
    "# 使用pre trained 模型对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "id": "da83ef84-16fd-4366-9a0e-3be21e731fa0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer,AutoModel\n",
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "id": "66d5dff5-b3eb-44d6-b871-926cda9de11a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "id": "ccf9a3e4-15c9-4233-aad4-c65c68c6e7d7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def mean_pooling(model_output, attention_mask):\n",
    "    token_embeddings = model_output[0].to(device) #First element of model_output contains all token embeddings\n",
    "    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float().to(device)\n",
    "    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "\n",
    "def load_model(model_location):    \n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location)\n",
    "   \n",
    "    model = AutoModel.from_pretrained(\n",
    "        model_location, \n",
    "        # device_map=\"balanced_low_0\", \n",
    "        #load_in_8bit=True\n",
    "    )\n",
    "    # load the model on GPU\n",
    "    model.to(device) \n",
    "    model.eval()\n",
    "    \n",
    "    return model, tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "id": "fedb25cf-5c33-41fd-8131-0c0c36cdebd5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model_location = 'sentence-transformers/paraphrase-multilingual-mpnet-base-v2'\n",
    "modelA,tokenizer  = load_model(model_location)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "id": "1f875dd8-c54c-4eb3-9516-9c9a7a16ae8e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "encoded_input = tokenizer(input_question, padding=True, truncation=True, max_length=128, return_tensors='pt').to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "id": "1d2ee9b2-e277-4e8e-90bd-6759665b091d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with torch.no_grad():\n",
    "    model_output = modelA(**encoded_input)\n",
    "\n",
    "# Perform pooling. In this case, max pooling.\n",
    "sentence_embeddings_q = mean_pooling(model_output, encoded_input['attention_mask']).to(device).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "id": "116a097b-ab2d-41d6-a2e8-309e5c27bf51",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "encoded_input = tokenizer(input_answer, padding=True, truncation=True, max_length=128, return_tensors='pt').to(device)\n",
    "with torch.no_grad():\n",
    "    model_output = modelA(**encoded_input)\n",
    "\n",
    "# Perform pooling. In this case, max pooling.\n",
    "sentence_embeddings_a = mean_pooling(model_output, encoded_input['attention_mask']).to(device).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "15726d48-4201-4edf-82ed-c0222afd82a5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "for i in range(len(input_question)):\n",
    "    sims = similarity(sentence_embeddings_a[i],sentence_embeddings_q[i])\n",
    "    print(f\"Question:{input_question[i]}\\nAnswer:{input_answer[i]}\\n{sims}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "id": "f29850ba-cfa6-476b-9a2b-8262aa55f22b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "input_queries = ['怎么攻击别人?']\n",
    "encoded_input = tokenizer(input_queries, padding=True, truncation=True, max_length=128, return_tensors='pt').to(device)\n",
    "with torch.no_grad():\n",
    "    model_output = modelA(**encoded_input)\n",
    "\n",
    "q_embedding = mean_pooling(model_output, encoded_input['attention_mask']).to(device).cpu().numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "id": "fa12cf2d-097c-4a0d-8725-d83b7df885fc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('您好，[援护]效果不分担燃烧伤害（除燃烧伤害外，都可以分担）。拥有点燃技能的英雄目前有寂，914，马克西莫，异化娜迪亚等。', '尤里卡突袭者的[援护]效果是否能分担各种英雄技能打出的伤害？') score:0.44886574149131775\n",
      "('玩家拥有建筑的占领权后，拥有【先锋】权限的玩家，可以开启对决。开启对决后，该玩家在此建筑中的所有部队都将会进入对决状态中，其他势力的拥有【先锋】权限的玩家，可以发起对决；当击败玩家所有进入对决状态的部队后，视为攻击成功。对决连续守住一定的攻击次数，或守住规定的时间，则对决成功，否则对决失败。对决获胜，可获得一定时间的无敌时间，并立即摧毁一定的城防值；对决失败，则击败对决的玩家获得建筑的控制权。', '先锋对决怎么玩，胜负规则是什么？') score:0.4420698285102844\n",
      "('您好，破浪突袭者BW-3的控制技能主要体现在 技能中有一项是潮汐震荡，如果被击目标处在攻击力降低状态，50%概率对其附加[眩晕]（无法普攻和释放主动技能）。从技能定位来看，其更偏向攻击，但防守效果也不错。', '小蓝是拥有控制技能的重装机兵，这个控制技能怎么理解？它更适合攻击还是防守？') score:0.43532323837280273\n"
     ]
    }
   ],
   "source": [
    "results = util.semantic_search(query_embeddings = q_embedding,corpus_embeddings= sentence_embeddings_a,top_k=3)\n",
    "results\n",
    "for ret in results[0]:\n",
    "    print(f\"{all_datas[ret['corpus_id']]} score:{ret['score']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f9fa8a3c-ce82-406c-b274-e1157838689c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'tokenizer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m input_queries \u001b[38;5;241m=\u001b[39m [\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m中国的首都在哪里？\u001b[39m\u001b[38;5;124m'\u001b[39m,\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m中国的首都在北京\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m----> 2\u001b[0m encoded_input \u001b[38;5;241m=\u001b[39m \u001b[43mtokenizer\u001b[49m(input_queries, padding\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, truncation\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, max_length\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m128\u001b[39m, return_tensors\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mpt\u001b[39m\u001b[38;5;124m'\u001b[39m)\u001b[38;5;241m.\u001b[39mto(device)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mno_grad():\n\u001b[1;32m      4\u001b[0m     model_output \u001b[38;5;241m=\u001b[39m modelA(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mencoded_input)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'tokenizer' is not defined"
     ]
    }
   ],
   "source": [
    "input_queries = ['中国的首都在哪里？','中国的首都在北京']\n",
    "encoded_input = tokenizer(input_queries, padding=True, truncation=True, max_length=128, return_tensors='pt').to(device)\n",
    "with torch.no_grad():\n",
    "    model_output = modelA(**encoded_input)\n",
    "\n",
    "embedding = mean_pooling(model_output, encoded_input['attention_mask']).to(device).cpu().numpy()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7cc99a0-ee6a-4902-b8af-b7c6cc9da18a",
   "metadata": {},
   "source": [
    "# 部署模型到sagemaker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57dc4832-b17a-47b0-8662-4ced85f29952",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "Requirement already satisfied: sagemaker in /opt/conda/lib/python3.10/site-packages (2.154.0)\n",
      "Collecting sagemaker\n",
      "  Downloading sagemaker-2.169.0.tar.gz (851 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m851.8/851.8 kB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25hCollecting attrs<24,>=23.1.0 (from sagemaker)\n",
      "  Using cached attrs-23.1.0-py3-none-any.whl (61 kB)\n",
      "Requirement already satisfied: boto3<2.0,>=1.26.131 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (1.26.132)\n",
      "Requirement already satisfied: cloudpickle==2.2.1 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (2.2.1)\n",
      "Requirement already satisfied: google-pasta in /opt/conda/lib/python3.10/site-packages (from sagemaker) (0.2.0)\n",
      "Requirement already satisfied: numpy<2.0,>=1.9.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (1.23.5)\n",
      "Requirement already satisfied: protobuf<4.0,>=3.1 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (3.20.3)\n",
      "Requirement already satisfied: protobuf3-to-dict<1.0,>=0.1.5 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (0.1.5)\n",
      "Requirement already satisfied: smdebug_rulesconfig==1.0.1 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (1.0.1)\n",
      "Requirement already satisfied: importlib-metadata<5.0,>=1.4.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (4.13.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (23.1)\n",
      "Requirement already satisfied: pandas in /opt/conda/lib/python3.10/site-packages (from sagemaker) (2.0.1)\n",
      "Requirement already satisfied: pathos in /opt/conda/lib/python3.10/site-packages (from sagemaker) (0.3.0)\n",
      "Requirement already satisfied: schema in /opt/conda/lib/python3.10/site-packages (from sagemaker) (0.7.5)\n",
      "Collecting PyYAML==6.0 (from sagemaker)\n",
      "  Using cached PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (682 kB)\n",
      "Requirement already satisfied: jsonschema in /opt/conda/lib/python3.10/site-packages (from sagemaker) (4.17.3)\n",
      "Requirement already satisfied: platformdirs in /opt/conda/lib/python3.10/site-packages (from sagemaker) (3.5.0)\n",
      "Requirement already satisfied: tblib==1.7.0 in /opt/conda/lib/python3.10/site-packages (from sagemaker) (1.7.0)\n",
      "Requirement already satisfied: botocore<1.30.0,>=1.29.132 in /opt/conda/lib/python3.10/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.29.132)\n",
      "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /opt/conda/lib/python3.10/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (1.0.1)\n",
      "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /opt/conda/lib/python3.10/site-packages (from boto3<2.0,>=1.26.131->sagemaker) (0.6.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.10/site-packages (from importlib-metadata<5.0,>=1.4.0->sagemaker) (3.15.0)\n",
      "Requirement already satisfied: six in /opt/conda/lib/python3.10/site-packages (from protobuf3-to-dict<1.0,>=0.1.5->sagemaker) (1.16.0)\n",
      "Requirement already satisfied: pyrsistent!=0.17.0,!=0.17.1,!=0.17.2,>=0.14.0 in /opt/conda/lib/python3.10/site-packages (from jsonschema->sagemaker) (0.19.3)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/lib/python3.10/site-packages (from pandas->sagemaker) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/conda/lib/python3.10/site-packages (from pandas->sagemaker) (2023.3)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /opt/conda/lib/python3.10/site-packages (from pandas->sagemaker) (2023.3)\n",
      "Requirement already satisfied: ppft>=1.7.6.6 in /opt/conda/lib/python3.10/site-packages (from pathos->sagemaker) (1.7.6.6)\n",
      "Requirement already satisfied: dill>=0.3.6 in /opt/conda/lib/python3.10/site-packages (from pathos->sagemaker) (0.3.6)\n",
      "Requirement already satisfied: pox>=0.3.2 in /opt/conda/lib/python3.10/site-packages (from pathos->sagemaker) (0.3.2)\n",
      "Requirement already satisfied: multiprocess>=0.70.14 in /opt/conda/lib/python3.10/site-packages (from pathos->sagemaker) (0.70.14)\n",
      "Requirement already satisfied: contextlib2>=0.5.5 in /opt/conda/lib/python3.10/site-packages (from schema->sagemaker) (21.6.0)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.25.4 in /opt/conda/lib/python3.10/site-packages (from botocore<1.30.0,>=1.29.132->boto3<2.0,>=1.26.131->sagemaker) (1.26.15)\n",
      "Building wheels for collected packages: sagemaker\n",
      "  Building wheel for sagemaker (setup.py) ... \u001b[?25ldone\n",
      "\u001b[?25h  Created wheel for sagemaker: filename=sagemaker-2.169.0-py2.py3-none-any.whl size=1158252 sha256=0ec721d00f4427b64ea6d9d4e052bb5989328fde366bb4acd16933fb90d67f82\n",
      "  Stored in directory: /root/.cache/pip/wheels/42/17/69/c2089332a0db669b4a27888e1d76e825168014112d5eb44231\n",
      "Successfully built sagemaker\n",
      "Installing collected packages: PyYAML, attrs, sagemaker\n",
      "  Attempting uninstall: PyYAML\n",
      "    Found existing installation: PyYAML 5.4.1\n",
      "    Uninstalling PyYAML-5.4.1:\n",
      "      Successfully uninstalled PyYAML-5.4.1\n",
      "  Attempting uninstall: attrs\n",
      "    Found existing installation: attrs 22.2.0\n",
      "    Uninstalling attrs-22.2.0:\n",
      "      Successfully uninstalled attrs-22.2.0\n",
      "  Attempting uninstall: sagemaker\n",
      "    Found existing installation: sagemaker 2.154.0\n",
      "    Uninstalling sagemaker-2.154.0:\n",
      "      Successfully uninstalled sagemaker-2.154.0\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "awscli 1.27.132 requires PyYAML<5.5,>=3.10, but you have pyyaml 6.0 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed PyYAML-6.0 attrs-23.1.0 sagemaker-2.169.0\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install -U sagemaker"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0490240-edff-4fff-8eec-bd295bf43f99",
   "metadata": {},
   "source": [
    "## 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "df4a6f16-63af-4c84-9dda-0af0dfa7b487",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "2fb24429-b2e5-4259-bd2d-70aa006c8cd1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "s3_code_prefix: LLM-RAG/workshop/finetuned-sentence2emb_deploy_code\n",
      "model_snapshot_path: ./finetuned-sentence-embedding\n"
     ]
    }
   ],
   "source": [
    "s3_model_prefix = \"LLM-RAG/workshop/finetuned-sentence2emb-model\"  # folder where model checkpoint will go\n",
    "model_snapshot_path = \"./finetuned-sentence-embedding\"\n",
    "s3_code_prefix = \"LLM-RAG/workshop/finetuned-sentence2emb_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "e53e83fd-305a-42f8-83cf-e0a17779eaac",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "upload: finetuned-sentence-embedding/config_sentence_transformers.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/config_sentence_transformers.json\n",
      "upload: finetuned-sentence-embedding/config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/config.json\n",
      "upload: finetuned-sentence-embedding/1_Pooling/config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/1_Pooling/config.json\n",
      "upload: finetuned-sentence-embedding/modules.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/modules.json\n",
      "upload: finetuned-sentence-embedding/README.md to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/README.md\n",
      "upload: finetuned-sentence-embedding/eval/.ipynb_checkpoints/similarity_evaluation_results-checkpoint.csv to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/eval/.ipynb_checkpoints/similarity_evaluation_results-checkpoint.csv\n",
      "upload: finetuned-sentence-embedding/special_tokens_map.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/special_tokens_map.json\n",
      "upload: finetuned-sentence-embedding/eval/similarity_evaluation_results.csv to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/eval/similarity_evaluation_results.csv\n",
      "upload: finetuned-sentence-embedding/sentence_bert_config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/sentence_bert_config.json\n",
      "upload: finetuned-sentence-embedding/tokenizer_config.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/tokenizer_config.json\n",
      "upload: finetuned-sentence-embedding/tokenizer.json to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/tokenizer.json\n",
      "upload: finetuned-sentence-embedding/sentencepiece.bpe.model to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/sentencepiece.bpe.model\n",
      "upload: finetuned-sentence-embedding/pytorch_model.bin to s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/pytorch_model.bin\n"
     ]
    }
   ],
   "source": [
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e4c4651-d731-4e8d-8deb-350918d8d721",
   "metadata": {},
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "8a07ff35-d547-4df3-b511-4be2862dcb6e",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\n"
     ]
    }
   ],
   "source": [
    "inference_image_uri = (\n",
    "    f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\"\n",
    ")\n",
    "\n",
    "#中国区需要替换为下面的image_uri\n",
    "# inference_image_uri = (\n",
    "#     f\"727897471807.dkr.ecr.{region}.amazonaws.com.cn/djl-inference:0.21.0-deepspeed0.8.3-cu117\"\n",
    "# )\n",
    "\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "4858bfe1-0392-41fd-8c46-d4a04a520397",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n"
     ]
    }
   ],
   "source": [
    "!mkdir -p sentence2emb_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "5fe26158-4b50-4659-a869-6d64f3d152e4",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting sentence2emb_deploy_code/model.py\n"
     ]
    }
   ],
   "source": [
    "%%writefile sentence2emb_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import torch\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "from transformers import pipeline, AutoModelForCausalLM, AutoTokenizer, AutoModel\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "print(f'--device={device}')\n",
    "\n",
    "\n",
    "def load_model(properties):\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "    \n",
    "    tokenizer = AutoTokenizer.from_pretrained(model_location)\n",
    "   \n",
    "    model = AutoModel.from_pretrained(\n",
    "        model_location, \n",
    "        # device_map=\"balanced_low_0\", \n",
    "        #load_in_8bit=True\n",
    "    )\n",
    "    # load the model on GPU\n",
    "    model.to(device) \n",
    "    model.requires_grad_(False)\n",
    "    model.eval()\n",
    "    \n",
    "    return model, tokenizer\n",
    "\n",
    "\n",
    "model = None\n",
    "tokenizer = None\n",
    "generator = None\n",
    "\n",
    "def mean_pooling(model_output, attention_mask):\n",
    "    token_embeddings = model_output[0].to(device) #First element of model_output contains all token embeddings\n",
    "    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float().to(device)\n",
    "    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)\n",
    "\n",
    "\n",
    "def handle(inputs: Input):\n",
    "    global model, tokenizer\n",
    "    if not model:\n",
    "        model, tokenizer = load_model(inputs.get_properties())\n",
    "\n",
    "    if inputs.is_empty():\n",
    "        return None\n",
    "    data = inputs.get_as_json()\n",
    "    \n",
    "    input_sentences = data[\"inputs\"]\n",
    "    params = data[\"parameters\"]\n",
    "    logging.info(f\"inputs: {input_sentences}\")\n",
    "    logging.info(f\"parameters: {params}\")\n",
    "    \n",
    "    encoded_input = tokenizer(input_sentences, padding=True, truncation=True, return_tensors='pt').to(device)\n",
    "    # Compute token embeddings\n",
    "    with torch.no_grad():\n",
    "        model_output = model(**encoded_input)\n",
    "\n",
    "    # Perform pooling. In this case, max pooling.\n",
    "    sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask']).to(device).cpu().numpy()\n",
    "\n",
    "#     # preprocess\n",
    "#     input_ids = tokenizer(input_sentences, return_tensors=\"pt\").input_ids\n",
    "#     # pass inputs with all kwargs in data\n",
    "#     if params is not None:\n",
    "#         outputs = model.generate(input_ids, **params)\n",
    "#     else:\n",
    "#         outputs = model.generate(input_ids)\n",
    "\n",
    "#     # postprocess the prediction\n",
    "#     prediction = tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "    \n",
    "    result = {\"sentence_embeddings\": sentence_embeddings}\n",
    "    return Output().add_as_json(result)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5c02fa9-d046-42a7-88fc-fc81fe890313",
   "metadata": {},
   "source": [
    "#### Note: option.s3url 需要按照自己的账号进行修改"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "64254ec0-69ab-4c2f-a9bb-05f259ab5218",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Overwriting sentence2emb_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "%%writefile sentence2emb_deploy_code/serving.properties\n",
    "engine=Python\n",
    "option.tensor_parallel_degree=1\n",
    "option.s3url = s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb-model/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "d76dad38-02f0-46f9-9121-9cb5155d7a76",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
      "To disable this warning, you can either:\n",
      "\t- Avoid using `tokenizers` before the fork if possible\n",
      "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
      "sentence2emb_deploy_code/\n",
      "sentence2emb_deploy_code/model.py\n",
      "sentence2emb_deploy_code/serving.properties\n"
     ]
    }
   ],
   "source": [
    "!rm s2e_model.tar.gz\n",
    "!cd sentence2emb_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf s2e_model.tar.gz sentence2emb_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "76d3d300-4d4d-4ea3-a7d8-2667619cd727",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S3 Code or Model tar ball uploaded to --- > s3://sagemaker-us-east-2-946277762357/LLM-RAG/workshop/finetuned-sentence2emb_deploy_code/s2e_model.tar.gz\n"
     ]
    }
   ],
   "source": [
    "s3_code_artifact = sess.upload_data(\"s2e_model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dc1a07e1-0955-497f-ba2d-9991a6deb833",
   "metadata": {},
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "b9318f47-36c1-4f16-962b-bdfe84c77c16",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finetuned-paraphrase-2023-07-05-03-14-23-508\n",
      "Image going to be used is ---- > 763104351884.dkr.ecr.us-east-2.amazonaws.com/djl-inference:0.21.0-deepspeed0.8.3-cu117\n",
      "Created Model: arn:aws:sagemaker:us-east-2:946277762357:model/finetuned-paraphrase-2023-07-05-03-14-23-508\n"
     ]
    }
   ],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(\"finetuned-paraphrase\") #Note: Need to specify model_name\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    "    \n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a7a2371-de79-4240-b029-80c15a674a7f",
   "metadata": {},
   "source": [
    "###  如果批量创建索引量较多，建议改成\"InstanceType\": \"ml.g4dn.xlarge\","
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "e337d531-3324-4ae4-ac76-66c3418fd548",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'EndpointConfigArn': 'arn:aws:sagemaker:us-east-2:946277762357:endpoint-config/finetuned-paraphrase-2023-07-05-03-14-23-508-config',\n",
       " 'ResponseMetadata': {'RequestId': '0b754ef3-6cb2-4243-bd14-518fbf35d080',\n",
       "  'HTTPStatusCode': 200,\n",
       "  'HTTPHeaders': {'x-amzn-requestid': '0b754ef3-6cb2-4243-bd14-518fbf35d080',\n",
       "   'content-type': 'application/x-amz-json-1.1',\n",
       "   'content-length': '132',\n",
       "   'date': 'Wed, 05 Jul 2023 03:14:30 GMT'},\n",
       "  'RetryAttempts': 0}}"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.m5.2xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 10*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "75f6a4b6-c36a-4ec5-8dd9-35b237ba55ba",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800000; text-decoration-color: #800000\">╭─────────────────────────────── </span><span style=\"color: #800000; text-decoration-color: #800000; font-weight: bold\">Traceback </span><span style=\"color: #bf7f7f; text-decoration-color: #bf7f7f; font-weight: bold\">(most recent call last)</span><span style=\"color: #800000; text-decoration-color: #800000\"> ────────────────────────────────╮</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/tmp/ipykernel_1225/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">4188345917.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">1</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">&lt;module&gt;</span>                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000; font-style: italic\">[Errno 2] No such file or directory: '/tmp/ipykernel_1225/4188345917.py'</span>                         <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/botocore/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">client.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">530</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_api_call</span>                      <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 527 │   │   │   │   │   </span><span style=\"color: #808000; text-decoration-color: #808000\">f\"{</span>py_operation_name<span style=\"color: #808000; text-decoration-color: #808000\">}() only accepts keyword arguments.\"</span>              <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 528 │   │   │   │   </span>)                                                                         <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 529 │   │   │   # The \"self\" in this scope is referring to the BaseClient.</span>                    <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 530 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>._make_api_call(operation_name, kwargs)                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 531 │   │   </span>                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 532 │   │   </span>_api_call.<span style=\"color: #ff0000; text-decoration-color: #ff0000\">__name__</span> = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">str</span>(py_operation_name)                                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 533 </span>                                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #bfbf7f; text-decoration-color: #bfbf7f\">/opt/conda/lib/python3.10/site-packages/botocore/</span><span style=\"color: #808000; text-decoration-color: #808000; font-weight: bold\">client.py</span>:<span style=\"color: #0000ff; text-decoration-color: #0000ff\">960</span> in <span style=\"color: #00ff00; text-decoration-color: #00ff00\">_make_api_call</span>                 <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>                                                                                                  <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 957 │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">if</span> http.status_code &gt;= <span style=\"color: #0000ff; text-decoration-color: #0000ff\">300</span>:                                                       <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 958 │   │   │   </span>error_code = parsed_response.get(<span style=\"color: #808000; text-decoration-color: #808000\">\"Error\"</span>, {}).get(<span style=\"color: #808000; text-decoration-color: #808000\">\"Code\"</span>)                     <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 959 │   │   │   </span>error_class = <span style=\"color: #00ffff; text-decoration-color: #00ffff\">self</span>.exceptions.from_code(error_code)                           <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span> <span style=\"color: #800000; text-decoration-color: #800000\">❱ </span> 960 <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\">│   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">raise</span> error_class(parsed_response, operation_name)                            <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 961 │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">else</span>:                                                                             <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 962 │   │   │   </span><span style=\"color: #0000ff; text-decoration-color: #0000ff\">return</span> parsed_response                                                        <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">│</span>   <span style=\"color: #7f7f7f; text-decoration-color: #7f7f7f\"> 963 </span>                                                                                          <span style=\"color: #800000; text-decoration-color: #800000\">│</span>\n",
       "<span style=\"color: #800000; text-decoration-color: #800000\">╰──────────────────────────────────────────────────────────────────────────────────────────────────╯</span>\n",
       "<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-weight: bold\">ClientError: </span>An error occurred <span style=\"font-weight: bold\">(</span>ValidationException<span style=\"font-weight: bold\">)</span> when calling the CreateEndpoint operation: Cannot create \n",
       "already existing endpoint \n",
       "<span style=\"color: #008000; text-decoration-color: #008000\">\"arn:aws:sagemaker:us-east-2:946277762357:endpoint/finetuned-paraphrase-2023-07-05-03-14-23-508-endpoint\"</span>.\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[31m╭─\u001b[0m\u001b[31m──────────────────────────────\u001b[0m\u001b[31m \u001b[0m\u001b[1;31mTraceback \u001b[0m\u001b[1;2;31m(most recent call last)\u001b[0m\u001b[31m \u001b[0m\u001b[31m───────────────────────────────\u001b[0m\u001b[31m─╮\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/tmp/ipykernel_1225/\u001b[0m\u001b[1;33m4188345917.py\u001b[0m:\u001b[94m1\u001b[0m in \u001b[92m<module>\u001b[0m                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[3;31m[Errno 2] No such file or directory: '/tmp/ipykernel_1225/4188345917.py'\u001b[0m                         \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/botocore/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m530\u001b[0m in \u001b[92m_api_call\u001b[0m                      \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 527 \u001b[0m\u001b[2m│   │   │   │   │   \u001b[0m\u001b[33mf\u001b[0m\u001b[33m\"\u001b[0m\u001b[33m{\u001b[0mpy_operation_name\u001b[33m}\u001b[0m\u001b[33m() only accepts keyword arguments.\u001b[0m\u001b[33m\"\u001b[0m              \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 528 \u001b[0m\u001b[2m│   │   │   │   \u001b[0m)                                                                         \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 529 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[2m# The \"self\" in this scope is referring to the BaseClient.\u001b[0m                    \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 530 \u001b[2m│   │   │   \u001b[0m\u001b[94mreturn\u001b[0m \u001b[96mself\u001b[0m._make_api_call(operation_name, kwargs)                            \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 531 \u001b[0m\u001b[2m│   │   \u001b[0m                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 532 \u001b[0m\u001b[2m│   │   \u001b[0m_api_call.\u001b[91m__name__\u001b[0m = \u001b[96mstr\u001b[0m(py_operation_name)                                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 533 \u001b[0m                                                                                          \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[2;33m/opt/conda/lib/python3.10/site-packages/botocore/\u001b[0m\u001b[1;33mclient.py\u001b[0m:\u001b[94m960\u001b[0m in \u001b[92m_make_api_call\u001b[0m                 \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m                                                                                                  \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 957 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94mif\u001b[0m http.status_code >= \u001b[94m300\u001b[0m:                                                       \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 958 \u001b[0m\u001b[2m│   │   │   \u001b[0merror_code = parsed_response.get(\u001b[33m\"\u001b[0m\u001b[33mError\u001b[0m\u001b[33m\"\u001b[0m, {}).get(\u001b[33m\"\u001b[0m\u001b[33mCode\u001b[0m\u001b[33m\"\u001b[0m)                     \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 959 \u001b[0m\u001b[2m│   │   │   \u001b[0merror_class = \u001b[96mself\u001b[0m.exceptions.from_code(error_code)                           \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m \u001b[31m❱ \u001b[0m 960 \u001b[2m│   │   │   \u001b[0m\u001b[94mraise\u001b[0m error_class(parsed_response, operation_name)                            \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 961 \u001b[0m\u001b[2m│   │   \u001b[0m\u001b[94melse\u001b[0m:                                                                             \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 962 \u001b[0m\u001b[2m│   │   │   \u001b[0m\u001b[94mreturn\u001b[0m parsed_response                                                        \u001b[31m│\u001b[0m\n",
       "\u001b[31m│\u001b[0m   \u001b[2m 963 \u001b[0m                                                                                          \u001b[31m│\u001b[0m\n",
       "\u001b[31m╰──────────────────────────────────────────────────────────────────────────────────────────────────╯\u001b[0m\n",
       "\u001b[1;91mClientError: \u001b[0mAn error occurred \u001b[1m(\u001b[0mValidationException\u001b[1m)\u001b[0m when calling the CreateEndpoint operation: Cannot create \n",
       "already existing endpoint \n",
       "\u001b[32m\"arn:aws:sagemaker:us-east-2:946277762357:endpoint/finetuned-paraphrase-2023-07-05-03-14-23-508-endpoint\"\u001b[0m.\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "277cc50a-d67e-4d01-ae58-18af86bf6259",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Status: InService\n",
      "Arn: arn:aws:sagemaker:us-east-2:946277762357:endpoint/finetuned-paraphrase-2023-07-04-16-08-10-303-endpoint\n",
      "Status: InService\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ef9ac01c-e8d9-4887-be4e-baded6d36abc",
   "metadata": {},
   "source": [
    "## 5. 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1bef7a45-2484-4c77-8b14-1f6d5fea6b01",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_vector_by_sm_endpoint(questions, sm_client, endpoint_name):\n",
    "    parameters = {\n",
    "        \"max_new_tokens\": 50,\n",
    "        \"temperature\": 0,\n",
    "        \"min_length\": 10,\n",
    "        \"no_repeat_ngram_size\": 2,\n",
    "    }\n",
    "\n",
    "    response_model = sm_client.invoke_endpoint(\n",
    "        EndpointName=endpoint_name,\n",
    "        Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": questions,\n",
    "                \"parameters\": parameters\n",
    "            }\n",
    "        ),\n",
    "        ContentType=\"application/json\",\n",
    "    )\n",
    "    json_str = response_model['Body'].read().decode('utf8')\n",
    "    json_obj = json.loads(json_str)\n",
    "    embeddings = json_obj['sentence_embeddings']\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "61b0cb25-1e16-4a70-a77c-f5e644d1166e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "prompts1 = \"\"\"专属技能碎片在哪里获得？\"\"\"\n",
    "prompts1 = \"\"\"中国首都在哪里？\"\"\"\n",
    "\n",
    "emb1 = get_vector_by_sm_endpoint(prompts1, smr_client, endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "2f37256a-7fa3-4e4c-b0c5-f17e0e8f858a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "prompts2 = \"\"\"专属技能碎片可以通过多种途径获得，例如礼包商城-特惠礼包界面可以购买专属技能碎片礼包\"\"\"\n",
    "emb2 = get_vector_by_sm_endpoint(prompts2, smr_client, endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "9377d20d-31c8-493e-9247-135052e35cbb",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0653]])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "util.cos_sim(emb1,emb2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f3a2f25-e206-439e-b8ee-b5f4216f09a9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.g4dn.xlarge",
  "kernelspec": {
   "display_name": "Python 3 (PyTorch 2.0.0 Python 3.10 GPU Optimized)",
   "language": "python",
   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-2:429704687514:image/pytorch-2.0.0-gpu-py310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}