{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "# FarOpt Basic Example Notebook\n", "\n", "This notebook shows how you can work with the FarOpt SDK. Full documentation can be found here https://faropt.readthedocs.io/" ] }, { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Requirement already satisfied: faropt==0.2.14 in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (0.2.14)\n", "Requirement already satisfied: boto3 in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (from faropt==0.2.14) (1.16.24)\n", "Requirement already satisfied: botocore<1.20.0,>=1.19.24 in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (from boto3->faropt==0.2.14) (1.19.24)\n", "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (from boto3->faropt==0.2.14) (0.3.3)\n", "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (from boto3->faropt==0.2.14) (0.10.0)\n", "Requirement already satisfied: urllib3<1.27,>=1.25.4; python_version != \"3.4\" in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (from botocore<1.20.0,>=1.19.24->boto3->faropt==0.2.14) (1.26.2)\n", "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (from botocore<1.20.0,>=1.19.24->boto3->faropt==0.2.14) (2.8.1)\n", "Requirement already satisfied: six>=1.5 in ./custom-miniconda/miniconda/envs/custom_faropt/lib/python3.7/site-packages (from python-dateutil<3.0.0,>=2.1->botocore<1.20.0,>=1.19.24->boto3->faropt==0.2.14) (1.15.0)\n" ] } ], "source": [ "import sys\n", "!{sys.executable} -m pip install --upgrade faropt" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "# !rm -rf /home/ec2-user/anaconda3/envs/JupyterSystemEnv/lib/python3.6/site-packages/faropt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Example OR tools script for vehicle routing:" ] }, { "cell_type": "code", "execution_count": 3, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "mkdir: cannot create directory ‘src’: File exists\r\n" ] } ], "source": [ "!mkdir src" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Overwriting src/main.py\n" ] } ], "source": [ "%%writefile src/main.py\n", "\n", "\"\"\"Capacited Vehicles Routing Problem (CVRP).\"\"\"\n", "\n", "# [START import]\n", "from __future__ import print_function\n", "from ortools.constraint_solver import routing_enums_pb2\n", "from ortools.constraint_solver import pywrapcp\n", "\n", "# Use this to publish custom metrics!\n", "from utils import *\n", "# [END import]\n", "\n", "\n", "# [START data_model]\n", "def create_data_model():\n", " \"\"\"Stores the data for the problem.\"\"\"\n", " data = {}\n", " data['distance_matrix'] = [\n", " [\n", " 0, 548, 776, 696, 582, 274, 502, 194, 308, 194, 536, 502, 388, 354,\n", " 468, 776, 662\n", " ],\n", " [\n", " 548, 0, 684, 308, 194, 502, 730, 354, 696, 742, 1084, 594, 480, 674,\n", " 1016, 868, 1210\n", " ],\n", " [\n", " 776, 684, 0, 992, 878, 502, 274, 810, 468, 742, 400, 1278, 1164,\n", " 1130, 788, 1552, 754\n", " ],\n", " [\n", " 696, 308, 992, 0, 114, 650, 878, 502, 844, 890, 1232, 514, 628, 822,\n", " 1164, 560, 1358\n", " ],\n", " [\n", " 582, 194, 878, 114, 0, 536, 764, 388, 730, 776, 1118, 400, 514, 708,\n", " 1050, 674, 1244\n", " ],\n", " [\n", " 274, 502, 502, 650, 536, 0, 228, 308, 194, 240, 582, 776, 662, 628,\n", " 514, 1050, 708\n", " ],\n", " [\n", " 502, 730, 274, 878, 764, 228, 0, 536, 194, 468, 354, 1004, 890, 856,\n", " 514, 1278, 480\n", " ],\n", " [\n", " 194, 354, 810, 502, 388, 308, 536, 0, 342, 388, 730, 468, 354, 320,\n", " 662, 742, 856\n", " ],\n", " [\n", " 308, 696, 468, 844, 730, 194, 194, 342, 0, 274, 388, 810, 696, 662,\n", " 320, 1084, 514\n", " ],\n", " [\n", " 194, 742, 742, 890, 776, 240, 468, 388, 274, 0, 342, 536, 422, 388,\n", " 274, 810, 468\n", " ],\n", " [\n", " 536, 1084, 400, 1232, 1118, 582, 354, 730, 388, 342, 0, 878, 764,\n", " 730, 388, 1152, 354\n", " ],\n", " [\n", " 502, 594, 1278, 514, 400, 776, 1004, 468, 810, 536, 878, 0, 114,\n", " 308, 650, 274, 844\n", " ],\n", " [\n", " 388, 480, 1164, 628, 514, 662, 890, 354, 696, 422, 764, 114, 0, 194,\n", " 536, 388, 730\n", " ],\n", " [\n", " 354, 674, 1130, 822, 708, 628, 856, 320, 662, 388, 730, 308, 194, 0,\n", " 342, 422, 536\n", " ],\n", " [\n", " 468, 1016, 788, 1164, 1050, 514, 514, 662, 320, 274, 388, 650, 536,\n", " 342, 0, 764, 194\n", " ],\n", " [\n", " 776, 868, 1552, 560, 674, 1050, 1278, 742, 1084, 810, 1152, 274,\n", " 388, 422, 764, 0, 798\n", " ],\n", " [\n", " 662, 1210, 754, 1358, 1244, 708, 480, 856, 514, 468, 354, 844, 730,\n", " 536, 194, 798, 0\n", " ],\n", " ]\n", " # [START demands_capacities]\n", " data['demands'] = [0, 1, 1, 2, 4, 2, 4, 8, 8, 1, 2, 1, 2, 4, 4, 8, 8]\n", " data['vehicle_capacities'] = [15, 15, 15, 15]\n", " # [END demands_capacities]\n", " data['num_vehicles'] = 4\n", " data['depot'] = 0\n", " return data\n", " # [END data_model]\n", "\n", "\n", "# [START solution_printer]\n", "def print_solution(data, manager, routing, solution):\n", " \"\"\"Prints solution on console.\"\"\"\n", " total_distance = 0\n", " total_load = 0\n", " for vehicle_id in range(data['num_vehicles']):\n", " index = routing.Start(vehicle_id)\n", " plan_output = 'Route for vehicle {}:\\n'.format(vehicle_id)\n", " route_distance = 0\n", " route_load = 0\n", " while not routing.IsEnd(index):\n", " node_index = manager.IndexToNode(index)\n", " route_load += data['demands'][node_index]\n", " plan_output += ' {0} Load({1}) -> '.format(node_index, route_load)\n", " previous_index = index\n", " index = solution.Value(routing.NextVar(index))\n", " route_distance += routing.GetArcCostForVehicle(\n", " previous_index, index, vehicle_id)\n", " plan_output += ' {0} Load({1})\\n'.format(manager.IndexToNode(index),\n", " route_load)\n", " plan_output += 'Distance of the route: {}m\\n'.format(route_distance)\n", " plan_output += 'Load of the route: {}\\n'.format(route_load)\n", " print(plan_output)\n", " total_distance += route_distance\n", " total_load += route_load\n", " print('Total distance of all routes: {}m'.format(total_distance))\n", " log_metric('total_distance',total_distance)\n", " save('/tmp/main.py') # or some other saved output\n", " print('Total load of all routes: {}'.format(total_load))\n", " # [END solution_printer]\n", "\n", "\n", "def main():\n", " \"\"\"Solve the CVRP problem.\"\"\"\n", " # Instantiate the data problem.\n", " # [START data]\n", " data = create_data_model()\n", " # [END data]\n", "\n", " # Create the routing index manager.\n", " # [START index_manager]\n", " manager = pywrapcp.RoutingIndexManager(len(data['distance_matrix']),\n", " data['num_vehicles'], data['depot'])\n", " # [END index_manager]\n", "\n", " # Create Routing Model.\n", " # [START routing_model]\n", " routing = pywrapcp.RoutingModel(manager)\n", "\n", " # [END routing_model]\n", "\n", " # Create and register a transit callback.\n", " # [START transit_callback]\n", " def distance_callback(from_index, to_index):\n", " \"\"\"Returns the distance between the two nodes.\"\"\"\n", " # Convert from routing variable Index to distance matrix NodeIndex.\n", " from_node = manager.IndexToNode(from_index)\n", " to_node = manager.IndexToNode(to_index)\n", " return data['distance_matrix'][from_node][to_node]\n", "\n", " transit_callback_index = routing.RegisterTransitCallback(distance_callback)\n", " # [END transit_callback]\n", "\n", " # Define cost of each arc.\n", " # [START arc_cost]\n", " routing.SetArcCostEvaluatorOfAllVehicles(transit_callback_index)\n", "\n", " # [END arc_cost]\n", "\n", " # Add Capacity constraint.\n", " # [START capacity_constraint]\n", " def demand_callback(from_index):\n", " \"\"\"Returns the demand of the node.\"\"\"\n", " # Convert from routing variable Index to demands NodeIndex.\n", " from_node = manager.IndexToNode(from_index)\n", " return data['demands'][from_node]\n", "\n", " demand_callback_index = routing.RegisterUnaryTransitCallback(\n", " demand_callback)\n", " routing.AddDimensionWithVehicleCapacity(\n", " demand_callback_index,\n", " 0, # null capacity slack\n", " data['vehicle_capacities'], # vehicle maximum capacities\n", " True, # start cumul to zero\n", " 'Capacity')\n", " # [END capacity_constraint]\n", "\n", " # Setting first solution heuristic.\n", " # [START parameters]\n", " search_parameters = pywrapcp.DefaultRoutingSearchParameters()\n", " search_parameters.first_solution_strategy = (\n", " routing_enums_pb2.FirstSolutionStrategy.PATH_CHEAPEST_ARC)\n", " # [END parameters]\n", "\n", " # Solve the problem.\n", " # [START solve]\n", " solution = routing.SolveWithParameters(search_parameters)\n", " # [END solve]\n", "\n", " # Print solution on console.\n", " # [START print_solution]\n", " print('printing solutions')\n", " if solution:\n", " print_solution(data, manager, routing, solution)\n", " # [END print_solution]\n", "\n", "\n", "main()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Additional note on utils included in the back end:\n", "\n", "Notice this line in the `print_solution()` function above:\n", "\n", "```\n", "log_metric('total_distance',total_distance)\n", "```\n", "\n", "This automatically pushes the float/int value as a named metric to cloudwatch logs that you can view after the job completes, or during the job." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Import and initialize a FarOpt object" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [], "source": [ "from faropt import FarOpt" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:FarOpt backend is ready!\n", "INFO:root:Async Bucket: faropt-s3asyncd0fda937-jk815bus3ob5\n", "INFO:root:Bucket: faropt-s3bucketfbfa637e-jauzfgds5cug\n", "INFO:root:Recipe Table: FaroptRecipeTable\n", "INFO:root:Job table: FaroptJobTable\n", "INFO:root:Lambda Opt function: faropt-lambdafunction2783057D4-4NK8UXFZPQBP\n" ] } ], "source": [ "fo = FarOpt()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ ".... _you should see \"INFO:root:FarOpt backend is ready!\" if the back end is set up correctly_" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Solve Vehicle routing using FarOpt" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Configure and submit the job" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "_This packages the source code (main.py and any subdirectories and files)_" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Listing project files ...\n", "INFO:root:Configured job!\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "./src/ [] ['main.py']\n", "main.py\n" ] } ], "source": [ "fo.configure('./src/')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "_....this can be a project that includes multiple folders and files, but requires a main.py at the root level_" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### submit() \n", "Submits to Fargate, and is suitable for arbitrarily long running jobs " ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Submitting job\n", "INFO:root:Submitted job! id: 2020-12-09-18-57-01-ced83a10-1ab9-4dc6-a4cc-6eb3f4fc1941\n" ] } ], "source": [ "fo.submit()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### You can check just the primary status of the job" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'PROVISIONING'" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fo.primary_status()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### .. or wait for job to complete\n", "\n", "..._look for INFO:root:JOB COMPLETED!_" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "PROVISIONING\n", "PROVISIONING\n", "PROVISIONING\n", "PROVISIONING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "RUNNING\n", "RUNNING\n", "RUNNING\n", "RUNNING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:JOB COMPLETED!\n" ] } ], "source": [ "fo.wait()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "_You should see ... INFO:root:JOB COMPLETED!_" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### View detailed logs from the job you ran" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "1607540265460 | Starting FarOpt backend\n", "1607540265460 | ███████╗ █████╗ ██████╗ ██████╗ ██████╗ ████████╗\n", "1607540265460 | ██╔════╝██╔══██╗██╔══██╗██╔═══██╗██╔══██╗╚══██╔══╝\n", "1607540265460 | █████╗ ███████║██████╔╝██║ ██║██████╔╝ ██║ \n", "1607540265460 | ██╔══╝ ██╔══██║██╔══██╗██║ ██║██╔═══╝ ██║ \n", "1607540265460 | ██║ ██║ ██║██║ ██║╚██████╔╝██║ ██║ \n", "1607540265460 | ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ \n", "1607540265460 | ---------------------------------------------------------------\n", "1607540265460 | Downloading source\n", "1607540265460 | Looking for source.zip in faropt-s3bucketfbfa637e-jauzfgds5cug / 2020-12-09-18-57-01-ced83a10-1ab9-4dc6-a4cc-6eb3f4fc1941\n", "1607540265460 | Note:\n", "1607540265460 | s3bucket : faropt-s3bucketfbfa637e-jauzfgds5cug\n", "1607540265460 | s3key : 2020-12-09-18-57-01-ced83a10-1ab9-4dc6-a4cc-6eb3f4fc1941\n", "1607540265460 | Downloaded source.zip! uncompressing\n", "1607540265460 | ---------------------------------------------------------------\n", "1607540265460 | ['main.py', 'utils.py', 'task.py']\n", "1607540265855 | printing solutions\n", "1607540265855 | Route for vehicle 0:\n", "1607540265855 | 0 Load(0) -> 1 Load(1) -> 4 Load(5) -> 3 Load(7) -> 15 Load(15) -> 0 Load(15)\n", "1607540265855 | Distance of the route: 2192m\n", "1607540265855 | Load of the route: 15\n", "1607540265855 | Route for vehicle 1:\n", "1607540265855 | 0 Load(0) -> 14 Load(4) -> 16 Load(12) -> 10 Load(14) -> 2 Load(15) -> 0 Load(15)\n", "1607540265855 | Distance of the route: 2192m\n", "1607540265855 | Load of the route: 15\n", "1607540265855 | Route for vehicle 2:\n", "1607540265855 | 0 Load(0) -> 7 Load(8) -> 13 Load(12) -> 12 Load(14) -> 11 Load(15) -> 0 Load(15)\n", "1607540265855 | Distance of the route: 1324m\n", "1607540265855 | Load of the route: 15\n", "1607540265855 | Route for vehicle 3:\n", "1607540265855 | 0 Load(0) -> 9 Load(1) -> 8 Load(9) -> 6 Load(13) -> 5 Load(15) -> 0 Load(15)\n", "1607540265855 | Distance of the route: 1164m\n", "1607540265855 | Load of the route: 15\n", "1607540265855 | Total distance of all routes: 6872m\n", "1607540265855 | {'ResponseMetadata': {'RequestId': 'f6506128-60a0-47e7-8dcf-35d3202517a3', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'f6506128-60a0-47e7-8dcf-35d3202517a3', 'content-type': 'text/xml', 'content-length': '212', 'date': 'Wed, 09 Dec 2020 18:57:45 GMT'}, 'RetryAttempts': 0}}\n", "1607540265855 | Saving /tmp/main.py to s3://faropt-s3bucketfbfa637e-jauzfgds5cug/2020-12-09-18-57-01-ced83a10-1ab9-4dc6-a4cc-6eb3f4fc1941\n", "1607540265855 | None\n", "1607540265855 | Total load of all routes: 60\n" ] } ], "source": [ "fo.logs()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "As you can see above, any output files that you save to /tmp will be automatically uploaded to the S3 bucket\n", "\n", "```Saving /tmp/output to s3:///path/```" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Optionally Add this problem as a standard recipe\n", "\n", "_This is useful when you need to repeatedly run the same problem, perhaps with different data inputs_" ] }, { "cell_type": "code", "execution_count": 15, "metadata": {}, "outputs": [], "source": [ "fo.add_recipe(recipe_name='cvrp_problem_v126',maintainer='Lab126')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Each recipe is given a unique ID, and this can be used to run the recipe at any time" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "str" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fo.get_recipe_id_from_description('cvrp_problem_v126')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Rerun this recipe at any time" ] }, { "cell_type": "code", "execution_count": 23, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Downloading recipe...\n", "INFO:root:Configured job!\n", "INFO:root:Submitting job\n", "INFO:root:Submitted job! id: 2020-12-09-19-18-58-3c4731c0-3dad-45ec-acdb-eae51c711ade\n" ] } ], "source": [ "fo.run_recipe(fo.get_recipe_id_from_description('cvrp_problem_v126'))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### ... Or directly run using the recipe ID" ] }, { "cell_type": "code", "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Downloading recipe...\n", "INFO:root:Configured job!\n", "INFO:root:Submitting job\n", "INFO:root:Submitted job! id: 2020-12-09-19-18-28-2dd6801e-52ef-43aa-986b-1faede889d20\n" ] } ], "source": [ "fo.run_recipe('e4191eda-b16e-4cea-80d1-abd5f80f75ad')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## You can also run this job as a micro job (on AWS Lambda)\n", "\n", "Note the the libraries you can use are limited to ortools, pyomo and deap libraries with default solvers\n" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Submitting job\n", "INFO:root:Staging job\n", "INFO:root:Staged job! id: 2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f\n", "INFO:root:Look for s3://faropt-s3asyncd0fda937-jk815bus3ob5/staged/2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f/source.zip\n", "INFO:root:By submitting a micro job, you are restricted to using ortools, pyomo and deap libraries for jobs that last up to 5 minutes\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "START RequestId: 8c9eb972-939c-4e86-83b5-6de4176eb527 Version: $LATEST\n", " Starting LambdaOpt backend\n", "\n", "███████╗ █████╗ ██████╗ ██████╗ ██████╗ ████████╗\n", "██╔════╝██╔══██╗██╔══██╗██╔═══██╗██╔══██╗╚══██╔══╝\n", "█████╗ ███████║██████╔╝██║ ██║██████╔╝ ██║ \n", "██╔══╝ ██╔══██║██╔══██╗██║ ██║██╔═══╝ ██║ \n", "██║ ██║ ██║██║ ██║╚██████╔╝██║ ██║ \n", "╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ \n", "\n", "\n", "---------------------------------------------------------------\n", "Downloading source\n", "Looking for source.zip in faropt-s3asyncd0fda937-jk815bus3ob5 / staged/2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f/source.zip\n", "Downloaded source.zip! uncompressing\n", "Setting env variables for micro environment\n", "---------------------------------------------------------------\n", "reformatting with black\n", "utils.py\n", "main.py\n", "printing solutions\n", "Route for vehicle 0:\n", " 0 Load(0) -> 1 Load(1) -> 4 Load(5) -> 3 Load(7) -> 15 Load(15) -> 0 Load(15)\n", "Distance of the route: 2192m\n", "Load of the route: 15\n", "\n", "Route for vehicle 1:\n", " 0 Load(0) -> 14 Load(4) -> 16 Load(12) -> 10 Load(14) -> 2 Load(15) -> 0 Load(15)\n", "Distance of the route: 2192m\n", "Load of the route: 15\n", "\n", "Route for vehicle 2:\n", " 0 Load(0) -> 7 Load(8) -> 13 Load(12) -> 12 Load(14) -> 11 Load(15) -> 0 Load(15)\n", "Distance of the route: 1324m\n", "Load of the route: 15\n", "\n", "Route for vehicle 3:\n", " 0 Load(0) -> 9 Load(1) -> 8 Load(9) -> 6 Load(13) -> 5 Load(15) -> 0 Load(15)\n", "Distance of the route: 1164m\n", "Load of the route: 15\n", "\n", "Total distance of all routes: 6872m\n", "{'ResponseMetadata': {'RequestId': '739991f1-2a66-40ad-ad47-7e61e8c40f5d', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': '739991f1-2a66-40ad-ad47-7e61e8c40f5d', 'content-type': 'text/xml', 'content-length': '212', 'date': 'Wed, 09 Dec 2020 19:19:49 GMT'}, 'RetryAttempts': 0}}\n", "Saving /tmp/main.py to s3://faropt-s3asyncd0fda937-jk815bus3ob5/staged/2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f\n", "None\n", "Total load of all routes: 60\n", "\n", "----\n", "None\n", "END RequestId: 8c9eb972-939c-4e86-83b5-6de4176eb527\n", "REPORT RequestId: 8c9eb972-939c-4e86-83b5-6de4176eb527\tDuration: 1082.15 ms\tBilled Duration: 1083 ms\tMemory Size: 2048 MB\tMax Memory Used: 149 MB\tInit Duration: 483.17 ms\t\n", "\n" ] } ], "source": [ "fo.submit(micro=True)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "_...or copy the recipe ID into the run_recipe call_" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### List past jobs and recipes" ] }, { "cell_type": "code", "execution_count": 25, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "jobid:2020-12-09-18-38-16-b9607d38-70fe-4986-8a6a-c5ee113770be | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-12-09-18-38-16-b9607d38-70fe-4986-8a6a-c5ee113770be/source.zip\n", "jobid:staged/2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f | bucket:faropt-s3asyncd0fda937-jk815bus3ob5 | path:staged/2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f/source.zip\n", "jobid:2020-12-09-19-18-58-3c4731c0-3dad-45ec-acdb-eae51c711ade | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-12-09-19-18-58-3c4731c0-3dad-45ec-acdb-eae51c711ade/source.zip\n", "jobid:staged/2020-12-09-18-28-22-2b742b17-a9f6-402b-95b8-8d11806c21d6 | bucket:faropt-s3asyncd0fda937-jk815bus3ob5 | path:staged/2020-12-09-18-28-22-2b742b17-a9f6-402b-95b8-8d11806c21d6/source.zip\n", "jobid:2020-12-09-18-20-24-783195c0-7c5c-4cbe-8c5e-6e3e40c0254d | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-12-09-18-20-24-783195c0-7c5c-4cbe-8c5e-6e3e40c0254d/source.zip\n", "jobid:2020-11-28-20-13-42-a6dbec89-b26a-456e-a523-03a0bc1a8890 | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-11-28-20-13-42-a6dbec89-b26a-456e-a523-03a0bc1a8890/source.zip\n", "jobid:2020-11-29-00-16-17-88fdd6a8-46e7-44d5-ae4e-5a040f2c6a78 | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-11-29-00-16-17-88fdd6a8-46e7-44d5-ae4e-5a040f2c6a78/source.zip\n", "jobid:2020-11-27-16-50-55-e479ecc9-91cb-4709-951a-750ba1f9621a | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-11-27-16-50-55-e479ecc9-91cb-4709-951a-750ba1f9621a/source.zip\n", "jobid:2020-12-09-18-12-00-fc1edc3a-098e-4271-9cba-d6fdf9c568b6 | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-12-09-18-12-00-fc1edc3a-098e-4271-9cba-d6fdf9c568b6/source.zip\n", "jobid:2020-11-28-22-04-31-68014b70-50db-426b-9dd4-c02c0e4824ab | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-11-28-22-04-31-68014b70-50db-426b-9dd4-c02c0e4824ab/source.zip\n" ] }, { "data": { "text/plain": [ "{'Items': [{'path': {'S': '2020-12-09-18-38-16-b9607d38-70fe-4986-8a6a-c5ee113770be/source.zip'},\n", " 'jobid': {'S': '2020-12-09-18-38-16-b9607d38-70fe-4986-8a6a-c5ee113770be'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}},\n", " {'path': {'S': 'staged/2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f/source.zip'},\n", " 'jobid': {'S': 'staged/2020-12-09-19-19-47-ef63ad97-c34d-4c25-a2d5-cebad5881e7f'},\n", " 'bucket': {'S': 'faropt-s3asyncd0fda937-jk815bus3ob5'}},\n", " {'path': {'S': '2020-12-09-19-18-58-3c4731c0-3dad-45ec-acdb-eae51c711ade/source.zip'},\n", " 'jobid': {'S': '2020-12-09-19-18-58-3c4731c0-3dad-45ec-acdb-eae51c711ade'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}},\n", " {'path': {'S': 'staged/2020-12-09-18-28-22-2b742b17-a9f6-402b-95b8-8d11806c21d6/source.zip'},\n", " 'jobid': {'S': 'staged/2020-12-09-18-28-22-2b742b17-a9f6-402b-95b8-8d11806c21d6'},\n", " 'bucket': {'S': 'faropt-s3asyncd0fda937-jk815bus3ob5'}},\n", " {'path': {'S': '2020-12-09-18-20-24-783195c0-7c5c-4cbe-8c5e-6e3e40c0254d/source.zip'},\n", " 'jobid': {'S': '2020-12-09-18-20-24-783195c0-7c5c-4cbe-8c5e-6e3e40c0254d'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}},\n", " {'path': {'S': '2020-11-28-20-13-42-a6dbec89-b26a-456e-a523-03a0bc1a8890/source.zip'},\n", " 'jobid': {'S': '2020-11-28-20-13-42-a6dbec89-b26a-456e-a523-03a0bc1a8890'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}},\n", " {'path': {'S': '2020-11-29-00-16-17-88fdd6a8-46e7-44d5-ae4e-5a040f2c6a78/source.zip'},\n", " 'jobid': {'S': '2020-11-29-00-16-17-88fdd6a8-46e7-44d5-ae4e-5a040f2c6a78'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}},\n", " {'path': {'S': '2020-11-27-16-50-55-e479ecc9-91cb-4709-951a-750ba1f9621a/source.zip'},\n", " 'jobid': {'S': '2020-11-27-16-50-55-e479ecc9-91cb-4709-951a-750ba1f9621a'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}},\n", " {'path': {'S': '2020-12-09-18-12-00-fc1edc3a-098e-4271-9cba-d6fdf9c568b6/source.zip'},\n", " 'jobid': {'S': '2020-12-09-18-12-00-fc1edc3a-098e-4271-9cba-d6fdf9c568b6'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}},\n", " {'path': {'S': '2020-11-28-22-04-31-68014b70-50db-426b-9dd4-c02c0e4824ab/source.zip'},\n", " 'jobid': {'S': '2020-11-28-22-04-31-68014b70-50db-426b-9dd4-c02c0e4824ab'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'}}],\n", " 'Count': 10,\n", " 'ScannedCount': 10,\n", " 'LastEvaluatedKey': {'jobid': {'S': '2020-11-28-22-04-31-68014b70-50db-426b-9dd4-c02c0e4824ab'}},\n", " 'ResponseMetadata': {'RequestId': 'UJE0N4BDH4MH446UVQ9S0ENC4VVV4KQNSO5AEMVJF66Q9ASUAAJG',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'server': 'Server',\n", " 'date': 'Wed, 09 Dec 2020 19:20:04 GMT',\n", " 'content-type': 'application/x-amz-json-1.0',\n", " 'content-length': '2280',\n", " 'connection': 'keep-alive',\n", " 'x-amzn-requestid': 'UJE0N4BDH4MH446UVQ9S0ENC4VVV4KQNSO5AEMVJF66Q9ASUAAJG',\n", " 'x-amz-crc32': '1220533782'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fo.list_jobs()" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "recipeid:e4191eda-b16e-4cea-80d1-abd5f80f75ad | bucket:faropt-s3bucketfbfa637e-jauzfgds5cug | path:2020-12-09-18-57-01-ced83a10-1ab9-4dc6-a4cc-6eb3f4fc1941/source.zip | description:cvrp_problem_v126 | maintainer:Lab126\n" ] }, { "data": { "text/plain": [ "{'Items': [{'recipeid': {'S': 'e4191eda-b16e-4cea-80d1-abd5f80f75ad'},\n", " 'code': {'S': 'see path'},\n", " 'maintainer': {'S': 'Lab126'},\n", " 'bucket': {'S': 'faropt-s3bucketfbfa637e-jauzfgds5cug'},\n", " 'description': {'S': 'cvrp_problem_v126'},\n", " 'path': {'S': '2020-12-09-18-57-01-ced83a10-1ab9-4dc6-a4cc-6eb3f4fc1941/source.zip'}}],\n", " 'Count': 1,\n", " 'ScannedCount': 1,\n", " 'ResponseMetadata': {'RequestId': 'K6MKNIMPU7B0SDC33AGP9B2AA7VV4KQNSO5AEMVJF66Q9ASUAAJG',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'server': 'Server',\n", " 'date': 'Wed, 09 Dec 2020 19:20:04 GMT',\n", " 'content-type': 'application/x-amz-json-1.0',\n", " 'content-length': '325',\n", " 'connection': 'keep-alive',\n", " 'x-amzn-requestid': 'K6MKNIMPU7B0SDC33AGP9B2AA7VV4KQNSO5AEMVJF66Q9ASUAAJG',\n", " 'x-amz-crc32': '2176190809'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fo.list_recipes()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Run a project directly from S3 (requires a source.zip in S3, with the main.py at the root)" ] }, { "cell_type": "code", "execution_count": 27, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:Downloading source...\n", "INFO:root:Configured job!\n", "INFO:root:Submitting job\n", "INFO:root:Submitted job! id: 2020-12-09-19-20-34-272c60c3-cfd9-4c7b-b657-868dfb5ae529\n" ] } ], "source": [ "fo.run_s3_job(bucket='faropt-s3bucketfbfa637e-jauzfgds5cug',key='2020-12-09-18-57-01-ced83a10-1ab9-4dc6-a4cc-6eb3f4fc1941/source.zip')" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "_You can also check Fargate service for running tasks!_" ] }, { "cell_type": "code", "execution_count": 28, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "PROVISIONING\n", "PROVISIONING\n", "PROVISIONING\n", "PROVISIONING\n", "PROVISIONING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "PENDING\n", "RUNNING\n", "RUNNING\n", "RUNNING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "DEPROVISIONING\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:JOB COMPLETED!\n" ] } ], "source": [ "fo.wait()" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "INFO:root:No running tasks. Checking completed tasks...\n", "INFO:root:No running tasks. Checking completed tasks...\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "1607541675806 | Starting FarOpt backend\n", "1607541675806 | ███████╗ █████╗ ██████╗ ██████╗ ██████╗ ████████╗\n", "1607541675806 | ██╔════╝██╔══██╗██╔══██╗██╔═══██╗██╔══██╗╚══██╔══╝\n", "1607541675806 | █████╗ ███████║██████╔╝██║ ██║██████╔╝ ██║ \n", "1607541675806 | ██╔══╝ ██╔══██║██╔══██╗██║ ██║██╔═══╝ ██║ \n", "1607541675806 | ██║ ██║ ██║██║ ██║╚██████╔╝██║ ██║ \n", "1607541675806 | ╚═╝ ╚═╝ ╚═╝╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ \n", "1607541675806 | ---------------------------------------------------------------\n", "1607541675806 | Downloading source\n", "1607541675806 | Looking for source.zip in faropt-s3bucketfbfa637e-jauzfgds5cug / 2020-12-09-19-20-34-272c60c3-cfd9-4c7b-b657-868dfb5ae529\n", "1607541675806 | Note:\n", "1607541675806 | s3bucket : faropt-s3bucketfbfa637e-jauzfgds5cug\n", "1607541675806 | s3key : 2020-12-09-19-20-34-272c60c3-cfd9-4c7b-b657-868dfb5ae529\n", "1607541675806 | Downloaded source.zip! uncompressing\n", "1607541675806 | ---------------------------------------------------------------\n", "1607541675806 | ['main.py', 'utils.py', 'task.py']\n", "1607541676192 | printing solutions\n", "1607541676192 | Route for vehicle 0:\n", "1607541676192 | 0 Load(0) -> 1 Load(1) -> 4 Load(5) -> 3 Load(7) -> 15 Load(15) -> 0 Load(15)\n", "1607541676192 | Distance of the route: 2192m\n", "1607541676192 | Load of the route: 15\n", "1607541676192 | Route for vehicle 1:\n", "1607541676192 | 0 Load(0) -> 14 Load(4) -> 16 Load(12) -> 10 Load(14) -> 2 Load(15) -> 0 Load(15)\n", "1607541676192 | Distance of the route: 2192m\n", "1607541676192 | Load of the route: 15\n", "1607541676192 | Route for vehicle 2:\n", "1607541676192 | 0 Load(0) -> 7 Load(8) -> 13 Load(12) -> 12 Load(14) -> 11 Load(15) -> 0 Load(15)\n", "1607541676192 | Distance of the route: 1324m\n", "1607541676192 | Load of the route: 15\n", "1607541676192 | Route for vehicle 3:\n", "1607541676192 | 0 Load(0) -> 9 Load(1) -> 8 Load(9) -> 6 Load(13) -> 5 Load(15) -> 0 Load(15)\n", "1607541676192 | Distance of the route: 1164m\n", "1607541676192 | Load of the route: 15\n", "1607541676192 | Total distance of all routes: 6872m\n", "1607541676192 | {'ResponseMetadata': {'RequestId': 'a5172421-3292-44b9-a7aa-98da1de15d50', 'HTTPStatusCode': 200, 'HTTPHeaders': {'x-amzn-requestid': 'a5172421-3292-44b9-a7aa-98da1de15d50', 'content-type': 'text/xml', 'content-length': '212', 'date': 'Wed, 09 Dec 2020 19:21:15 GMT'}, 'RetryAttempts': 0}}\n", "1607541676192 | Saving /tmp/main.py to s3://faropt-s3bucketfbfa637e-jauzfgds5cug/2020-12-09-19-20-34-272c60c3-cfd9-4c7b-b657-868dfb5ae529\n", "1607541676192 | None\n", "1607541676192 | Total load of all routes: 60\n" ] } ], "source": [ "fo.logs()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Getting metrics" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Using utils.py, we logged a metric called \"total_distance\" in while submitting the job. Let's get the publised values for this metric, for this job." ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'Label': 'total_distance',\n", " 'Datapoints': [{'Timestamp': datetime.datetime(2020, 12, 9, 19, 21, tzinfo=tzlocal()),\n", " 'Average': 6872.0,\n", " 'Minimum': 6872.0,\n", " 'Maximum': 6872.0,\n", " 'Unit': 'None'}],\n", " 'ResponseMetadata': {'RequestId': '400a53f7-a863-4b1b-82e5-3e95a27d375d',\n", " 'HTTPStatusCode': 200,\n", " 'HTTPHeaders': {'x-amzn-requestid': '400a53f7-a863-4b1b-82e5-3e95a27d375d',\n", " 'content-type': 'text/xml',\n", " 'content-length': '565',\n", " 'date': 'Wed, 09 Dec 2020 19:28:11 GMT'},\n", " 'RetryAttempts': 0}}" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "fo.get_metric_data(metric_name='total_distance')" ] } ], "metadata": { "kernelspec": { "display_name": "conda_custom_faropt", "language": "python", "name": "conda_custom_faropt" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.7.8" } }, "nbformat": 4, "nbformat_minor": 4 }