+ + POD_NAME=tensorflow-benchmarks-efa-worker-0POD_NAME=tensorflow-benchmarks-efa-worker-1 + + shiftshift + + /opt/kube/kubectl/opt/kube/kubectl exec exec tensorflow-benchmarks-efa-worker-0 tensorflow-benchmarks-efa-worker-1 -- -- /bin/sh /bin/sh -c -c orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 2 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 1 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=tensorflow-benchmarks-efa-worker-2 + shift + /opt/kube/kubectl exec tensorflow-benchmarks-efa-worker-2 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 3 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=tensorflow-benchmarks-efa-worker-4 + shift + /opt/kube/kubectl exec tensorflow-benchmarks-efa-worker-4 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 5 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=tensorflow-benchmarks-efa-worker-3 + shift + + POD_NAME=tensorflow-benchmarks-efa-worker-7/opt/kube/kubectl exec+ tensorflow-benchmarks-efa-worker-3shift -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 4 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" + /opt/kube/kubectl exec tensorflow-benchmarks-efa-worker-7 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 8 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=tensorflow-benchmarks-efa-worker-5 + shift + /opt/kube/kubectl exec tensorflow-benchmarks-efa-worker-5 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 6 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=tensorflow-benchmarks-efa-worker-6 + shift + /opt/kube/kubectl exec tensorflow-benchmarks-efa-worker-6 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "3972726784" -mca ess_base_vpid 7 -mca ess_base_num_procs "9" -mca orte_node_regex "tensorflow-benchmarks-efa-launcher-[1:7]xtnq,tensorflow-benchmarks-efa-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "3972726784.0;tcp://192.168.41.237:47035" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_mapping_policy "slot" -mca pmix "^s1,s2,cray,isolated" WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/compat/v2_compat.py:88: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version. Instructions for updating: non-resource variables are not supported in the long term 2020-07-09 23:21:36.105462: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105548: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105631: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105579: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105590: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105606: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105538: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105612: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105647: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105788: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105701: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105807: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105783: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105779: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105852: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105873: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105953: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105995: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105892: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105953: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106088: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105823: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106025: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106000: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106033: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105970: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106139: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106141: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.105992: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106085: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106133: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106041: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106228: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106117: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106005: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106239: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106073: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106268: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106091: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106289: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106104: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106323: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106292: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106353: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106376: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106314: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106326: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106366: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106421: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106347: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106324: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106553: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106369: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106388: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106562: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106496: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106470: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106529: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106401: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106490: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106594: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106553: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106593: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.106660: I tensorflow/core/platform/cpu_feature_guard.cc:142] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX512F 2020-07-09 23:21:36.111466: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.111465: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.111466: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.111584: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x590f110 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.111602: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.111585: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4a07550 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.111604: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.111610: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56f0500 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.111630: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.111756: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.111863: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4fc0120 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.111883: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.111859: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.111859: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.111989: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5177c80 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.111987: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x46f8b20 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112017: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112016: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112017: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112061: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112079: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112157: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4730030 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112179: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112171: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x42f8fc0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112189: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112218: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x448b620 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112248: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112222: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112278: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112402: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4385e60 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112384: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x48e72f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112404: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112426: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112497: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112453: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112500: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112608: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x51743f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112628: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112617: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x45ce0a0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112638: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112619: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x54bfa30 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112638: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112814: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112823: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.112938: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56e94c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112935: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4f2e550 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.112956: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.112958: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113040: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113040: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113043: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113040: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113041: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113042: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113162: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4459720 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113181: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113158: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x536bee0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113176: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113158: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4e1e1c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113175: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113184: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4e1f2f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113204: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113178: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x47e3c60 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113198: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113182: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x59a6650 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113198: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113257: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113360: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5306cc0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113381: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.113571: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.113692: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5892a10 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.113711: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.114137: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114138: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114458: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114470: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114503: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114672: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114481: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114596: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5a5c020 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.114740: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114503: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114672: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114490: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114601: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x42ea9b0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.114673: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114513: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114778: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114479: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114596: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x50feaa0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.114618: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.114842: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114513: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114654: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4198640 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.114675: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.114618: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115032: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114650: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4772910 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.114671: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115023: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.114620: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.114642: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5a52330 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.114661: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.114637: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x52b4310 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.114654: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.114806: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.114988: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x419d100 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115010: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115100: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.115093: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.115336: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.115390: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.115568: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115561: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115639: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115629: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115640: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115721: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5aabb80 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115743: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115641: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115744: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x59b96d0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115763: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115536: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115640: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115536: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115716: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115537: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115776: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x57988f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115794: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115550: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115784: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4099900 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115800: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115836: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115714: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4aa6ac0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115734: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115772: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x517a5c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115792: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115882: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4941920 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115920: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115756: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4f5dd90 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115775: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115790: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x48c0260 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115811: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115798: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x40db4c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115818: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115847: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x522e600 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115867: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115812: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115851: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115801: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x46eb3c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115822: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115814: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.115870: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.116016: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x572ba80 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116038: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115878: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.115949: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x52d0cc0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.115823: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.115981: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3f2c750 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116018: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116072: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115883: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.115972: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115816: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.115998: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5402660 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116020: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115889: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.115976: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115820: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.116095: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115899: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.116136: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115814: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.116204: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5702b80 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116222: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116209: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3d592e0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116232: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115899: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.116132: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x419b850 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116155: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.115919: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.116028: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4fec420 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116049: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116185: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.115938: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.116031: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5300c00 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116053: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116332: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.116038: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3d91d60 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116059: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116050: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x52ee720 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116071: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116278: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x432e010 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116300: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116272: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.116068: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x44af5c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116089: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116353: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x52f3a50 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116372: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116216: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.116355: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4cf4750 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116378: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116456: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x57ab7f0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116479: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116429: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.116545: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.116580: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x48d9e90 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116603: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116668: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5bb9210 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116690: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116653: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2500000000 Hz 2020-07-09 23:21:36.116750: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2499995000 Hz 2020-07-09 23:21:36.116789: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x442b640 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116812: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.116872: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x59f86e0 initialized for platform Host (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:36.116894: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version 2020-07-09 23:21:36.117232: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.117235: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.117230: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.117231: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.117233: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.117264: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.117379: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.117404: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118448: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118443: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118444: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118572: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118445: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118571: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118504: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118445: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118570: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118497: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118448: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118574: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118500: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118459: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118573: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118505: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118698: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118831: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118762: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118501: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118765: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118589: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118764: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118664: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118760: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118757: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118761: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.118978: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.119050: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.119316: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.119336: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:36.120229: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcuda.so.1 2020-07-09 23:21:38.403456: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.403440: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.403473: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.403489: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.403467: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.403473: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.403502: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.404410: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.404468: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.404486: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.404467: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.404492: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.404494: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.413802: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.413805: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.413794: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.413910: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.415579: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3ee2030 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.415608: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.415655: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3eb1280 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.415683: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.415679: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5361dc0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.415703: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.415686: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4a32fc0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.415717: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.415762: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x439b290 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.415783: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.415898: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x530ab30 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.415922: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.416574: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x50a3e90 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.416600: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.416683: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x29f2bf0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.416707: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.416750: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5043fa0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.416775: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.416819: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x446a300 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.416843: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.417081: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4429d30 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.417123: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.417108: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4cc1a10 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.417130: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.417218: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3db7610 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.417240: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.417723: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.417971: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418052: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418070: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418163: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418296: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418385: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418657: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418733: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418832: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.418909: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.419386: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.419276: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.419543: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.421993: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x50c0730 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.422024: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.422140: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x527b470 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.422162: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.422201: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x42c5620 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.422223: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.422310: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x44d8260 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.422334: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.423360: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.423868: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.423947: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.424025: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.424120: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.424377: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.424438: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.424812: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x41d6e30 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.424836: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.426281: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.429400: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.429483: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.429763: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.429829: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.429936: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.429999: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.430019: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.430096: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.430120: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.430184: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.430200: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.430258: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.431452: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.431549: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.431618: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.431598: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.431697: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.431699: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.431765: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.431788: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.431778: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.431805: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.431832: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.431875: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.431866: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.431931: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.433095: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.433159: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.433291: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4fdb130 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.433317: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.433410: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x413c900 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.433436: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.433505: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.433516: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.433518: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.433516: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.433518: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.433613: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.433515: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.433615: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.433615: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.433739: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.433687: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.433819: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.433775: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.433956: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.434016: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.433906: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.434038: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.433912: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.434103: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.433917: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.434119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.433914: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.434188: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.433921: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.433909: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.433997: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.434033: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.434480: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.434645: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.434551: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.434614: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.434614: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.435284: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.435282: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.435460: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.435281: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.435522: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.435553: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.435582: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.435582: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.435583: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.435616: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.435579: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.435581: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.435697: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.435864: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.435994: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.435990: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.436013: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.436001: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.435961: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.436131: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.435959: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.435959: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.436125: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5b1aa50 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.436165: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.435954: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.435961: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.436151: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.435959: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.436417: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4a380d0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.436449: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.436587: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.436783: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.436974: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.437127: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.437206: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.437212: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.437213: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.437207: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.437221: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.437209: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.437219: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.437209: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.437220: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.437212: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.437460: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.437386: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.437458: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.437470: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.437521: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.437590: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.437602: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.437595: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.437608: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.437764: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.437730: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.438008: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.438002: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.438012: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.438010: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.438162: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.438231: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.438507: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.438551: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.438686: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.438743: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.438786: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.438811: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.439134: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.439109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.439175: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.439136: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.439137: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.439135: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.439202: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.439313: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.439312: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.439656: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.439662: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.439655: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.439656: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.439656: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.440214: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.440384: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.440461: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.440795: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4214ce0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.440822: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.440775: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.440942: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.440776: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.440888: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4a02640 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.440924: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.440785: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.440940: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x42cb6b0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.440968: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.441051: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5712c10 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.441079: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.440776: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.440992: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x51b25d0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.441017: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.440785: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.441076: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.441079: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.441352: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441443: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.441409: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.441371: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.441510: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.441416: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.441380: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.441380: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.441370: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.441370: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.441520: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441747: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441808: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.441380: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.441416: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.441541: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441544: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441542: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441539: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441541: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441901: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.441899: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.442342: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.442625: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.442669: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.442837: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.442907: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.442990: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.443285: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.443277: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.443461: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.443323: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.443282: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.443396: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.443320: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.443284: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.443395: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.443400: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.443407: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.443519: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.443625: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.443998: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.444064: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.444450: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.444430: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.444443: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.444720: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.444773: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.444778: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.444792: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.444788: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.444787: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.444792: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.444793: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.444790: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.444929: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.444929: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.444929: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.444930: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.444929: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.445373: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.445373: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.445498: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.445495: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.445764: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.445809: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.446407: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.446474: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.447120: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5b21dc0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.447145: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.447189: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5497bc0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.447211: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.447480: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.447360: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.447423: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.447808: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.447788: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.447850: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.447889: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.447860: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.447956: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.447973: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448294: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.448274: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.448273: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.448380: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448401: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448737: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448817: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448698: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448777: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448715: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448712: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.448724: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.449213: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.449498: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.449502: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.449565: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.449733: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.449856: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.449799: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.450086: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.450650: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.450894: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.450897: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.450911: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.451265: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.451261: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.451275: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.451724: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.452267: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5a5b2a0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.452291: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.452427: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x467a760 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.452456: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.452540: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.452630: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.452800: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.452820: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.452865: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.452966: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.452988: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x49c1700 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.453018: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.453079: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5698f90 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.453101: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.453093: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.453160: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.453191: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.453170: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.453381: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.453443: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.453556: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.453613: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.454085: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454275: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.454270: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.454275: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.454472: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.454428: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454561: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454582: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454343: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454364: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454336: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454332: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454345: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454340: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454545: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454760: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454835: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.454954: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.455132: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.455319: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.455304: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.455308: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.455304: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.455308: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.456269: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.456638: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.456742: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.457197: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.457197: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.457550: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.457549: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.457747: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x57ddca0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.457784: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.457790: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.457804: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.457864: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.457904: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.457942: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.458309: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458393: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458350: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458469: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458310: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.458312: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.458555: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458324: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.458419: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458419: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458427: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458637: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458661: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458718: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.458746: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.459266: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.459232: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.459241: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.459562: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.459561: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.460611: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.460612: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.460458: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.460636: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.461147: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4359270 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.461175: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.461277: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.461387: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x58c1ef0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.461354: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.461439: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.461507: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4cd3890 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.461458: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.461533: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.461626: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x56a5b20 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.461652: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.461696: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x512ba90 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.461722: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.461800: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.461857: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.462666: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.463062: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.463295: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x467e410 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.463327: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.463984: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.463956: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2547a10 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.463985: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.464385: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.464477: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x45d3dd0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.464502: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.464725: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.464728: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.464810: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.464815: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.464843: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.465891: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.466723: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.466788: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.466894: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.466955: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.466860: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.466981: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.466882: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.467039: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.466878: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.467061: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.466884: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.467116: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.466885: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.466871: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.467158: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.466887: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.466981: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.467203: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.467243: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.467140: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x592fe10 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467176: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.467306: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.467354: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x479bd50 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467388: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.467474: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4e523b0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467506: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.467597: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4422480 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467626: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.467699: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x49e8d40 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467697: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x52c6bc0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467724: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.467816: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3d3aa10 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467841: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.467724: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.467844: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4e57af0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.467874: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.468096: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4862280 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.468116: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.468437: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x50a5800 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.468463: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.469564: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.469635: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.469654: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.469726: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.469760: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.469829: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.469768: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.469840: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.469916: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.470029: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.470111: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.470121: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.470286: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.470345: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.470280: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.470387: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.470239: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.470472: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.470514: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.470557: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.470609: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.470545: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.470722: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.470776: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.470806: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.470862: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.470820: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.471418: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.471534: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.471606: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.471604: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.471601: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.471610: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.471646: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.471943: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.471982: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.472149: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.472973: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.473009: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x595a450 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.473031: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.473253: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.473261: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.473215: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.473299: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.473529: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.473642: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.473643: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.473647: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.474028: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.474031: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.474030: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.474138: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5241c20 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.474169: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.474203: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x48fae60 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.474235: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.474480: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.475001: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.474981: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.475289: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.475527: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.475646: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.475772: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.475815: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.475852: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.475900: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.475955: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.475934: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.475984: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476018: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3db17e0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.476042: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.476029: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.476031: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.476020: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.476030: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.476072: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476139: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x59da2a0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.476163: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.476235: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.476310: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476316: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.476381: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476491: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.476566: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476583: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.476703: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.476742: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.476664: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476803: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476776: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.476826: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.476878: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.477036: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.477205: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.477204: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.477204: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.477511: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.477591: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.477995: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.478067: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.477710: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.477833: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.478149: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.478232: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.478446: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.478274: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.478341: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.478570: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.478727: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.478665: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.478700: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.478859: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.478803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.478920: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.479423: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.479465: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.479527: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.479465: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.479580: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.479790: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.479797: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.479846: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.479873: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.480153: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.480382: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.480738: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.481251: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x487c5e0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.481277: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.481514: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x40a73d0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.481541: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.481565: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.481642: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x413ce50 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.481668: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.481565: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.481622: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x513f630 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.481658: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.481563: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.481682: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.481678: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.481679: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.481807: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x3342820 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.481695: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.481815: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5499c70 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.481843: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.481696: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.481834: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.481697: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.481914: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5337310 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices: 2020-07-09 23:21:38.481905: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.481941: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla V100-SXM2-32GB, Compute Capability 7.0 2020-07-09 23:21:38.481844: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.481949: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.482703: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.482767: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.482760: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.482761: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.482964: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.482946: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.483024: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.483804: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.483856: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.483936: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.483943: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484006: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484106: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.484185: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.484261: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.484204: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484349: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.484273: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484289: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484444: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.484353: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484376: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484374: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484460: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484453: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484467: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484554: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484555: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484632: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484638: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.484694: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484758: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484731: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484827: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484809: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484840: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.484924: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.484910: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.485134: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.485214: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.485592: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.485681: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.485911: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.485922: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486059: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.485922: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.485951: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.486001: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.486000: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486053: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.486017: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.486030: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486224: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486344: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486405: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486460: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.486556: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.486601: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486461: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.486459: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.486526: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.486706: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.486585: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.486586: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.486575: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.486647: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.486886: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.486879: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.487201: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.487254: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.487449: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.487451: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487652: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487615: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.487792: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.487766: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.487867: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.487949: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.487955: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487931: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487933: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.488116: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487937: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487934: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.488254: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.487936: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487932: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.487934: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.488345: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488428: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488355: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488499: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.488362: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488354: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488351: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488359: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488355: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488781: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.488806: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.488880: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.488965: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.488919: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.488966: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.489049: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.489062: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.489150: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.489214: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.489543: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.489629: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.489854: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.489966: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490139: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490168: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.490252: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.490255: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.490331: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.490398: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490403: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490398: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490403: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490493: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490622: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.490405: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490674: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.490406: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490409: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.490919: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.491044: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491169: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491525: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.491550: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491591: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.491572: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491691: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.491576: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491759: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.491578: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491584: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491825: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.491585: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491583: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.491581: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.492047: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.491978: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.492014: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.492115: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.492136: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.492218: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.492868: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.493144: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.493449: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.493456: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.493746: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.493816: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.493806: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.493765: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.493923: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.494341: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.494340: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.494453: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.494448: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.494588: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.494701: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.494831: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.494950: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.495085: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.495176: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.495221: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.495235: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.495235: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.495507: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.495510: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.495548: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.495926: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.495925: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.496161: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.495918: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.495893: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.495901: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.495914: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.495901: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.495920: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.496017: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.496056: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.496053: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.496043: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.496017: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.496039: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.496052: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.496567: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.496044: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.497200: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.497221: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.497745: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.498200: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.498288: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.498717: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.498810: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.498922: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.498982: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.499015: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.499084: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.499102: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.499166: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.499186: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.499255: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.499354: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.499413: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.499438: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:38.499508: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.499644: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.499764: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.500307: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.500412: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.500635: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.500643: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.500873: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.500867: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.500873: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.501026: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.501114: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:38.501346: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.501428: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.501517: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.501701: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.501726: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.501797: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.501843: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.501863: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.501892: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.502001: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.502149: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.502210: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.502426: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.502503: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.502426: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.502456: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.502570: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.502459: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.502455: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.502457: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.502458: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:38.502830: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.502830: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.502879: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.502877: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.502877: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.502882: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.502875: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:38.502911: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.504816: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.504814: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.504976: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.504976: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.504985: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.504980: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.504982: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:38.505563: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.505677: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.505780: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.505912: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.505913: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.506107: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.506103: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.506108: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.506110: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.506110: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:38.506259: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.506339: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.506554: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.507605: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.508053: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.509717: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.509800: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.509879: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.509969: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510049: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510120: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510208: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510138: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.510137: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.510241: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510235: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510343: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510359: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.510359: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.510357: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.510359: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.510359: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:38.510506: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510502: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510504: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510503: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.510505: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.515301: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.515379: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.516478: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.516536: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.516785: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.516841: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.516873: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.516929: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.517140: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.517197: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.517465: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.517516: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.518207: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.518268: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.518156: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.518253: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.521546: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.521908: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.521967: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.522073: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.522938: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.522861: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.523038: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.523126: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.523201: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.523871: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.523939: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.523997: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.524059: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524080: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.524140: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524115: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.524177: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524233: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.524304: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524283: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.524337: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524367: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.524450: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.524422: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524516: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524458: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.524521: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524539: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.524593: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524622: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.524678: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524706: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.524762: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.524812: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.524871: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.532921: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.532971: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.533183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.533293: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.533945: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.534004: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.534640: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:38.534695: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.535303: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:38.535374: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.535382: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:38.535435: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.535470: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:38.535534: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.535569: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:38.535643: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.535758: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:38.535821: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.536234: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:38.536283: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.536325: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:38.536376: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:38.677361: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.677416: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.677426: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.677765: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.679568: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.681261: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.702941 139767071725376 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.704804 139767071725376 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.733294 139767071725376 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.734297: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.734350: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.734360: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:38.734331: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.734369: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.734378: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:38.734669: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.734697: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.734736: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.734744: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.734763: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.735086: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.735138: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.735168: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.735175: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:38.735267: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.735300: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.735308: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.735695: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.735752: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.735784: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.735791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:38.735884: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.736361: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.736462: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.736491: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.736499: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:38.736685: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.736713: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.736720: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:38.737057: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.738360: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.744931: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.746395: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.748216: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.750470: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.750898: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.751449: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.752321: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.753419: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.758069: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.758121: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.758129: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:38.758436: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.758470: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.758471: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.758480: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:38.758828: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.760120: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:38.760324: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.760378: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.760386: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.760741: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.761311: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:38.762327: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.762646: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:38.763335: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.764620: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:38.764940: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:38.765194: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:38.765533: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:38.765617: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:38.766186: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.766231: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.766240: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.766540: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.766741: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.766779: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.766787: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N Generating training model 2020-07-09 23:21:38.767119: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.767071: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.768466: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.768498: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.768506: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N Generating training model 2020-07-09 23:21:38.768964: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.768978: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.769610: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.770198: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 2020-07-09 23:21:38.771148: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model 2020-07-09 23:21:38.771486: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod Generating training model ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model 2020-07-09 23:21:38.773837: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.773899: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.773908: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:38.774081: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:38.774211: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.774279: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.775355: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.775416: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.775425: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.775707: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.775739: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.775746: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:38.775828: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 TensorFlow: 2.1 Model: resnet101 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.776068: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.776070: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) Generating training model Generating training model 2020-07-09 23:21:38.776511: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.776607: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.776650: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.776658: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:38.776647: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.776688: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.776696: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:38.776983: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.777015: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.777022: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:38.777111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.777142: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.777149: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:38.777146: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.777262: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.777330: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.777363: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.777370: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:38.777420: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.777451: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.777459: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.778085: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.778119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.778126: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.778136: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.778238: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.778258: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.778285: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.778293: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.778272: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.778280: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N Generating training model 2020-07-09 23:21:38.778411: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:38.778488: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.778599: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.778574: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.778630: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.778638: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:38.778701: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.778664: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.778788: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.779157: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.779690: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 2020-07-09 23:21:38.780397: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.780442: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.780451: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:38.780378: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.780443: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.780452: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.780806: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.780713 139719730001728 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.780885: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero Generating training model 2020-07-09 23:21:38.781240: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.781275: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.781282: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.781561: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.781616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.781625: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:38.781706: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.782119: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero W0709 23:21:38.781956 140605020944192 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] Use `tf.keras.layers.Conv2D` instead. NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.782233 139719730001728 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.782620: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.782779: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.783062: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.783099: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.783107: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.783181: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.783301 140605020944192 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.783887 140691758417728 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.784231 140598316468032 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Generating training model Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.784468: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.784858: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.784930: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.784939: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:38.785319: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.785344: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.785388: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.785396: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:38.785620: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.785655: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.785662: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.785427 140691758417728 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.785684: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.785662 140598316468032 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.785756 139982119135040 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.786003: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.785767: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.785806: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.785817: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.786072 140347868399424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.786453 140512030914368 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.786747 139898123876160 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.787041: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.787076: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.787084: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:38.787067: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.787118: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.787126: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:38.787135: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787246: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.787253 139982119135040 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.787311: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787392: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787413: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787415: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787416: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787644: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.787696: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787642: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.787671: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero W0709 23:21:38.787557 140347868399424 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.787679: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.787687: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.787879: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.787908: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.787915: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N W0709 23:21:38.787866 140512030914368 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.787880: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.787927: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.787935: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:38.788015: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.787958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.787998: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.788006: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.788081: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788110: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.788117: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:38.788162: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788197: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.788206: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.788277: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788308: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.788316: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.788242 139898123876160 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.788596: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788629: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.788743: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.788637: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:38.788709: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788738: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.788853: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.788746: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:38.788893: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788748: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788775: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.788782: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:38.788923: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.788931: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:38.788791: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788816: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.788823: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:38.788937: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.788941: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.788842: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.788958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.788988: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.788996: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:38.789019: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.789105: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.789209: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.789240: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.789247: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:38.789291: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.789366: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.789395: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.789390: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.789402: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.789568: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.789477: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.789816: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.789832: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.790162: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.790076: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.790109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.790117: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:38.790096: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.790127: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.790135: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:38.790178: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.790264 139769147275072 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.790460 139954201352000 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.791282: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.791341: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:38.791413: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:38.791399: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.791606: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.791607: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:38.791782: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.791970 139769147275072 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.792098 139954201352000 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.792808: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero W0709 23:21:38.792727 140712519374656 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.793483: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.793714: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.794055: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.794333 140712519374656 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.794438 140159716185920 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.794613: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.794662: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.794740: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.794995: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:38.794992: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.795085: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.795167: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:38.795244: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.795237 140021899769664 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.795539: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.795928 140159716185920 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. Generating training model WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.796622 140021899769664 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 64 per device Num batches: 100 Num epochs: 0.32 W0709 23:21:38.797810 140033300801344 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model 2020-07-09 23:21:38.798050: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:38.798190: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.798137: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.799204 140033300801344 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.799262: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.800074: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:38.800523: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:38.800781: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training Generating training model SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 2020-07-09 23:21:38.801558: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.801806: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.801861: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero Generating training model 2020-07-09 23:21:38.802687: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.802793: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.803133: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.803315: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.803340: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:38.803417: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:38.803562: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.803647: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.803839: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.804340: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.804680: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.804698: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) Generating training model 2020-07-09 23:21:38.805548: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:38.805455: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.805996: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:38.806071: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:38.806066: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:38.806161: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:38.806126: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:38.806328: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:38.806334: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:38.806424: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.806390 139719730001728 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.806657: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Generating training model Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Optimizer: sgd Variables: horovod ========== Generating training model Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.807453 140605020944192 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.808159: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.808770 139648303425344 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.809664 140691758417728 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.809975 140598316468032 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.810243: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. TensorFlow: 2.1 W0709 23:21:38.810225 139648303425344 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.810879 139630126221120 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.810918 140605867497280 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.811057: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:38.811355: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:38.811525: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:38.811611: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.811576 140562346370880 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.811634 140347868399424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.811851: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.811865 140512030914368 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.811953: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.812112 139982119135040 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Use keras.layers.MaxPooling2D instead. TensorFlow: 2.1 TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Generating training model WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.812513 140605867497280 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.812514 139630126221120 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Generating training model Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model W0709 23:21:38.812888 139898123876160 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. TensorFlow: 2.1 TensorFlow: 2.1 2020-07-09 23:21:38.813089: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) W0709 23:21:38.813075 140562346370880 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Generating training model Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.814476 140422864979776 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.814655: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.814717: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:38.814726: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:38.815062: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.814896: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:38.815145: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:38.815300: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.815247 139829151631168 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.815454 140339437655872 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.815889 140422864979776 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.815972: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:38.816134: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:38.816103: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.816160: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:38.816169: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:38.816306: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False 2020-07-09 23:21:38.816471: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:38.816592: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.816545: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training TensorFlow: 2.1 Batch size: 4096 global 64 per device Num batches: 100 WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.816477 139769147275072 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 SingleSess: False Batch size: 4096 global WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 64 per device Num batches: 100 Num epochs: 0.32 W0709 23:21:38.816711 139829151631168 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.816726: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== ========== 2020-07-09 23:21:38.816803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.816777 139954201352000 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. TensorFlow: 2.1 Generating training model Generating training model Generating training model WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 W0709 23:21:38.817006 140339437655872 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training 2020-07-09 23:21:38.817491: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.817450 140690824218432 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Num epochs: 0.32 Use `tf.keras.layers.Conv2D` instead. Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model TensorFlow: 2.1 WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.818807 140712519374656 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.818896 140690824218432 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.819920 140159716185920 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.820466: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.820465 140435199129408 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.820828 140021899769664 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.820800 140136554874688 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.820764 139728809436992 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.821961: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.822027 140435199129408 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.822301 139728809436992 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.822394 140136554874688 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Generating training model Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] Generating training model W0709 23:21:38.823668 139982497134400 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== 2020-07-09 23:21:38.823746: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.824004 140543473829696 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.824064 140033300801344 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== TensorFlow: 2.1 Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.824170 139889036683072 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global Generating training model 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.825093 139982497134400 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.825094 140576162572096 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.825654 140543473829696 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.825676 139889036683072 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.825939 139793959765824 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.826150 139806201362240 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.826193 140111135848256 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.826421 140576162572096 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.826451 139810623559488 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.826725 140496823912256 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.826854 140336345724736 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.827105 140511694276416 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.827289 139793959765824 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.827653 139806201362240 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.827668 140111135848256 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.828060 139810623559488 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.828322 140496823912256 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.828434 140336345724736 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.828775 140511694276416 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== W0709 23:21:38.829140 139795966580544 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Generating training model Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.829548 140445836597056 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.829790: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.829650 140659616442176 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.829826: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:38.829834: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.829928 140345154484032 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.830119: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.830398 140058100578112 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.830636 139834293708608 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.830808 139795966580544 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.831048 140659616442176 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.830930 140445836597056 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.831309 140345154484032 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.831833: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.831840 140058100578112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.832145 139834293708608 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.832844 139761894754112 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.833546: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.833893 139648303425344 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.834367 139761894754112 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.835811 140448709056320 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.836167 139766263830336 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.836554 139661611177792 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.836559 140595026159424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.836841 140605867497280 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.836759 140101099308864 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.837079 139630126221120 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.837269 140562346370880 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead.W0709 23:21:38.837255 140448709056320 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.837170 140439388542784 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.837200 139781685286720 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.837322 139853248595776 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.837798 139766263830336 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.837817 139738826934080 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.838074 139661611177792 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.837917 140595026159424 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.837849 139958812215104 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.838087 140101099308864 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.838035 140079197558592 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.838357 139935576172352 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.838771 140439388542784 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.838760 139781685286720 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.839532 139853248595776 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.839421 139958812215104 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.839419 139738826934080 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.839660 140079197558592 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.839861 140422864979776 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.840686 139935576172352 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.841317 139829151631168 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.841746 140339437655872 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.843041: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.843087: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:38.843095: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.843033 140690824218432 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.843417: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.843666 139684024837952 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.843839: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.843871: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:38.843879: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N W0709 23:21:38.843796 140154930235200 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. 2020-07-09 23:21:38.844194: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.844227: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:38.844235: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:38.844192: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.844226: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:38.844235: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:38.844899: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.844892: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:38.844922: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:38.844930: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:38.844963: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.845077: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.845342: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero W0709 23:21:38.845287 139684024837952 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.845362 140154930235200 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.846142 140435199129408 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.846509 139728809436992 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.846884 140136554874688 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.847102: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.848970 139982497134400 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.850089 140543473829696 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.850242 139889036683072 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.850673 140576162572096 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.850783 139793959765824 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.851463 140111135848256 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.851948 139806201362240 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.852200 140544370415424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.852515 140496823912256 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.852836: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero W0709 23:21:38.852883 140336345724736 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.852961 139810623559488 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.853585 140511694276416 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.853651 140544370415424 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. 2020-07-09 23:21:38.853781: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.854134: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:38.854202: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.854882 140445836597056 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.855013 140659616442176 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.855031 140345154484032 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.855798 140058100578112 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.856003: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.856353 139834293708608 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.858930 139761894754112 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.860843: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.860672 139795966580544 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. 2020-07-09 23:21:38.861011: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:38.861190: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:38.861267: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.861460 140448709056320 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.861916 139661611177792 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.861886 140101099308864 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.862127 140595026159424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.862918 139766263830336 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.863016 139781685286720 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.863152 140439388542784 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.864179 139958812215104 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.864274 139738826934080 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.864330 140079197558592 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== ========== Generating training model Generating training model TensorFlow: 2.1 Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod TensorFlow: 2.1 ========== Model: resnet101 Dataset: imagenet (synthetic) Mode: training SingleSess: False Batch size: 4096 global 64 per device Num batches: 100 Num epochs: 0.32 Devices: ['horovod/gpu:0', 'horovod/gpu:1', 'horovod/gpu:2', 'horovod/gpu:3', 'horovod/gpu:4', 'horovod/gpu:5', 'horovod/gpu:6', 'horovod/gpu:7', 'horovod/gpu:8', 'horovod/gpu:9', 'horovod/gpu:10', 'horovod/gpu:11', 'horovod/gpu:12', 'horovod/gpu:13', 'horovod/gpu:14', 'horovod/gpu:15', 'horovod/gpu:16', 'horovod/gpu:17', 'horovod/gpu:18', 'horovod/gpu:19', 'horovod/gpu:20', 'horovod/gpu:21', 'horovod/gpu:22', 'horovod/gpu:23', 'horovod/gpu:24', 'horovod/gpu:25', 'horovod/gpu:26', 'horovod/gpu:27', 'horovod/gpu:28', 'horovod/gpu:29', 'horovod/gpu:30', 'horovod/gpu:31', 'horovod/gpu:32', 'horovod/gpu:33', 'horovod/gpu:34', 'horovod/gpu:35', 'horovod/gpu:36', 'horovod/gpu:37', 'horovod/gpu:38', 'horovod/gpu:39', 'horovod/gpu:40', 'horovod/gpu:41', 'horovod/gpu:42', 'horovod/gpu:43', 'horovod/gpu:44', 'horovod/gpu:45', 'horovod/gpu:46', 'horovod/gpu:47', 'horovod/gpu:48', 'horovod/gpu:49', 'horovod/gpu:50', 'horovod/gpu:51', 'horovod/gpu:52', 'horovod/gpu:53', 'horovod/gpu:54', 'horovod/gpu:55', 'horovod/gpu:56', 'horovod/gpu:57', 'horovod/gpu:58', 'horovod/gpu:59', 'horovod/gpu:60', 'horovod/gpu:61', 'horovod/gpu:62', 'horovod/gpu:63'] NUMA bind: False Data format: NCHW Optimizer: sgd Variables: horovod ========== Generating training model Generating training model WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.869705 139684024837952 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.869997 140154930235200 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.875443 140458393626432 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.876369 139853248595776 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.877006 140458393626432 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.878023 139935576172352 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead.W0709 23:21:38.878044 140544370415424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.880359 140571334534976 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.880932 139655422154560 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.881392 139979895777088 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.881601 140493844432704 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:134: conv2d (from tensorflow.python.layers.convolutional) is deprecated and will be removed in a future version. Instructions for updating: Use `tf.keras.layers.Conv2D` instead. W0709 23:21:38.881689 140571334534976 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.882354 139655422154560 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.882872 139979895777088 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. W0709 23:21:38.883095 140493844432704 deprecation.py:323] From /usr/local/lib/python3.6/dist-packages/tensorflow_core/python/layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer) is deprecated and will be removed in a future version. Instructions for updating: Please use `layer.__call__` method instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.901469 140458393626432 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.905881 140571334534976 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.906685 139979895777088 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.906635 139655422154560 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. W0709 23:21:38.907250 140493844432704 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/convnet_builder.py:266: max_pooling2d (from tensorflow.python.layers.pooling) is deprecated and will be removed in a future version. Instructions for updating: Use keras.layers.MaxPooling2D instead. Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph Initializing graph WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.698639 139648303425344 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.707066 140605867497280 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.708679 139793959765824 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.714719 140159716185920 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.729401 140111135848256 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.733777 140512030914368 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.735691 139767071725376 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.739206 140033300801344 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession Initializing graph WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.748852 140058100578112 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.749825 140345154484032 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.751574 140347868399424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.757153 140021899769664 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.757936 139982497134400 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.759689 140691758417728 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.760142 139661611177792 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.761965 139719730001728 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.763115 140101099308864 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.766776 140605020944192 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.770246 140659616442176 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.770887 139954201352000 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.779922 139769147275072 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.780655 140496823912256 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.781444 139728809436992 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.784906 140543473829696 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.785732 139684024837952 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.787824 140422864979776 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.788604 139898123876160 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.793136 139979895777088 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.796528 140712519374656 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.797101 140079197558592 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.802225 139834293708608 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.803227 140576162572096 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.805328 139781685286720 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.807524 140439388542784 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.811994 139806201362240 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.812837 139829151631168 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.813568 140598316468032 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.816217 140562346370880 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.826986 140136554874688 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.837520 140690824218432 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.841587 139761894754112 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.842507 140154930235200 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.844903 140448709056320 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.845556 140336345724736 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.845514 139630126221120 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.845749 140339437655872 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.852298 140544370415424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.852405 140511694276416 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.852460 140595026159424 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.853421 139795966580544 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.854256 139982119135040 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.855977 139738826934080 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.856439 139889036683072 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.862670 140493844432704 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.866382 139958812215104 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.876795 139655422154560 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.888947 140571334534976 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.897822 139853248595776 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.900484 139810623559488 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.910407 140458393626432 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.950803 139935576172352 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:42.972174 139766263830336 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:43.104933 140435199129408 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession WARNING:tensorflow:From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession W0709 23:21:43.126309 140445836597056 deprecation.py:323] From /workspace/benchmarks/scripts/tf_cnn_benchmarks/benchmark_cnn.py:2268: Supervisor.__init__ (from tensorflow.python.training.supervisor) is deprecated and will be removed in a future version. Instructions for updating: Please switch to tf.train.MonitoredTrainingSession 2020-07-09 23:21:43.324302: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.326068: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.326115: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.326141: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.326152: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.326162: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.326173: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.326183: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.326193: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.326250: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.326954: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.327954: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.328715: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.328764: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.328799: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.328809: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.328820: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.328829: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.328839: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.328849: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.328901: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.329638: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.329667: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.329674: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.329680: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.329820: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.330572: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.331504: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.332258: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.332288: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.332295: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.332300: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.332440: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.333148: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.334110: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.335771: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.339068: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.340836: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.340886: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.340915: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.340926: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.340937: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.340963: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.340973: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.340985: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.341062: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.342794: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.344422: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.344450: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.344458: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.344463: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.344597: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.346279: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.347926: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.353543: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.355315: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.355364: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.355395: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.355406: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.355416: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.355426: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.355435: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.355445: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.355498: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.357155: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.358787: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.358818: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.358825: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.358831: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.358967: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.360628: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.362486: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.370210: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.370843: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.372044: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.372106: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.372141: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.372156: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.372168: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.372181: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.372194: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.372206: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.372269: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.372608: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.372693: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.372744: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.372777: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.372788: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.372799: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.372809: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.372819: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.372829: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.372879: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.373963: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.375591: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.375624: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.375632: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.375638: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.375777: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.376063: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.376111: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.376137: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.376148: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.376158: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.376169: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.376179: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.376190: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.376247: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.376335: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.377469: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.379111: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.379633: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.379664: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.379696: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.379703: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.379709: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.379878: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.379901: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.381650: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.381735: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.381777: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.381791: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.381804: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.381817: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.381830: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.381844: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.381912: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.381920: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.383021: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.383052: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.383059: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.383065: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.383239: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.383321: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.383708: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.383580: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.383756: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.383788: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.383800: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.383810: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.383821: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.383831: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.383841: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.383893: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.384577: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.385219: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.385252: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.385259: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.385266: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.385404: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.385562: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.387201: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.387143: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.387124: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.387236: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.387243: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.387249: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.387184: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:43.387385: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.388794: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.389090: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.388977: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.389033: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.389074: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.389088: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.389101: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.389113: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.389126: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.389138: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.389195: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.390744: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.390883: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.392058: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.393719: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.393754: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.393761: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.393767: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.393907: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.395566: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.395396: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.396024: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.397210: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.397154: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.397197: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.397228: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.397239: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.397250: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.397259: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.397269: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.397279: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.397328: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.397803: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.397866: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.397902: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.397917: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.397930: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.397944: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.397957: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.397970: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.398032: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.398396: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.398855: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.399015: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.399709: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.400157: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.400208: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.400239: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.400250: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.400260: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.400270: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.400280: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.400291: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.400352: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.400645: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.400674: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.400680: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.400686: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.400651: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.400704: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.400739: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.400750: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.400761: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.400771: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.400781: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.400820: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.400792: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.400874: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.401420: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.401454: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.401461: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.401467: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.401611: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.402069: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.402076: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.402494: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.402574: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.403269: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.404199: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.404212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.404245: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.404252: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.404258: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.404405: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.404935: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.405658: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.405697: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.405704: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.405710: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.405754: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.405802: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.405832: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.405843: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.405854: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.405864: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.405874: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.405887: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.405900: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.405944: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.406117: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.407830: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.409323: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.409434: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.409505: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.410274: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.411600: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.411660: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.411696: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.411707: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.411718: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.411729: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.411739: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.411750: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.411823: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.412997: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.413063: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.413096: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.413104: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.413109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.413288: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.413334: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.413362: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.413250: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.413376: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.413386: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.413397: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.413407: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.413417: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.413471: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.414155: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.415030: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.414906: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.415298: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.416546: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.416521: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.416575: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.416610: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.416621: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.416632: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.416642: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.416652: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.416663: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.416721: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.416717: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.416645: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.418295: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.418366: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.418350: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.418387: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.418395: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.418401: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.418406: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.418420: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.418433: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.418447: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.418460: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.418474: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.418530: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.418404: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.418450: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.418480: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.418492: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.418503: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.418513: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.418524: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.418534: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.418588: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.419335: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.420109: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.420145: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.420153: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.420159: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.420091: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.420329: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.420253: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.421923: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.422035: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.421874: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.421905: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.421911: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.421917: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.422049: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.423200: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.423629: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.423670: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.423677: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.423684: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.423790: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.424093: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.424564: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.425426: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.425514: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.425549: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.425556: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.425561: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.426432: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.426616: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.426997: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:43.427685: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.428343: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.428405: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.428434: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.428445: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.428455: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.428465: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.428475: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.428485: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.428557: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.428701: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.429767: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.430382: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.431538: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.431590: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.431620: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.431631: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.431649: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.431660: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.431671: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.431682: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.431751: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.431610: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.432035: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.432069: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.432076: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.432082: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.432006: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.432063: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.432097: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.432223: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.432108: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.432119: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.432130: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.432140: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.432151: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.432210: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.432915: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.433423: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.433884: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.434811: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.435082: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.435122: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.435129: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.435136: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.435289: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.435535: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.435492: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.435768: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.436570: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.436630: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.436684: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.436700: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.436713: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.436726: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.436738: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.436752: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.436819: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.437027: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.437423: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.437456: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.437463: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.437468: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.437617: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.438492: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.438683: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.438857: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.439283: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.439906: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.440192: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.440223: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.440230: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.440235: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.440377: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.440630: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.440685: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.440717: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.440729: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.440739: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.440749: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.440759: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.440770: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.440840: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.440965: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.441612: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.441663: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.441693: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.441704: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.441728: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.441739: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.441749: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.441759: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.441839: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.442128: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.442536: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.443520: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.443796: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.444173: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.444204: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.444211: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.444218: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.444355: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.445173: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.445204: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.445211: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.445217: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.445376: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.446049: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.447128: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.447755: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.448765: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.449213: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.450965: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.451013: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.451043: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.451053: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.451064: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.451074: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.451084: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.451094: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.451161: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.452898: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.454529: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.454563: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.454570: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.454576: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.454713: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.454950: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.456390: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.456469: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.456730: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.456781: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.456813: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.456824: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.456834: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.456844: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.456854: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.456865: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.456919: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.458028: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.458269: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.458328: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.458357: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.458369: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.458380: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.458391: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.458401: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.458412: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.458485: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.458579: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.460209: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.460284: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.460318: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.460325: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.460332: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.460476: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.461848: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.461881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.461887: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.461893: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.462032: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.462140: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.462102: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.463270: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.463707: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.463802: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.464001: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.464357: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.464421: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.464460: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.464471: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.464481: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.464492: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.464501: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.464512: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.464592: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.465220: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.465348: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.465713: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.465778: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.465820: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.465833: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.465846: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.465859: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.465872: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.465886: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.465954: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.466041: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.466090: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.466125: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.466147: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.466158: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.466168: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.466178: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.466188: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.466242: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.466306: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.467016: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.467079: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.467122: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.467136: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.467149: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.467161: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.467173: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.467186: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.467242: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.467640: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.467760: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.468051: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.468108: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.468140: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.468152: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.468162: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.468173: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.468183: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.468194: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.468247: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.468935: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.469313: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.469345: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.469352: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.469358: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.469505: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.469446: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.469598: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.469911: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.470562: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.470596: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.470603: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.470609: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.470744: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.471110: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.471147: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.471155: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.471161: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.471212: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.471595: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.471507: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.471570: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.471606: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.471630: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.471638: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.471644: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.471618: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.471628: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.471638: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.471648: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.471659: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.471732: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.471788: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.472057: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.472511: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.472823: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.472855: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.472862: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.472868: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.472856: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.473041: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.472884: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.473455: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.473486: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.474183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.474653: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.474704: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.474739: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.474750: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.474761: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.474772: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.474703: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.474785: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.474796: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.474849: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.475130: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.475120: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.475157: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.475164: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.475169: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.475321: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.476400: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.476526: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.477002: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.477115: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.478047: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.478185: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.478219: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.478226: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.478232: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.478375: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.479271: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.480104: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.480274: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.480435: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.480495: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.480529: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.480541: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.480551: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.480563: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.480573: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.480584: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.480647: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.481749: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.482367: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.484075: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.484110: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.484117: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.484124: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.484265: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.485941: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.486728: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.487600: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.488480: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:17.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.488537: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.488564: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.488576: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.488587: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.488598: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.488609: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.488620: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.488691: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.490359: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.492057: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 1 2020-07-09 23:21:43.492091: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.492099: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 1 2020-07-09 23:21:43.492105: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 1: N 2020-07-09 23:21:43.492248: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.493936: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.494157: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.494541: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.495585: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 1, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:17.0, compute capability: 7.0) 2020-07-09 23:21:43.495938: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.495997: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.496027: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.496038: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.496049: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.496059: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.496069: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.496080: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.496152: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.496308: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.496365: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.496407: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.496421: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.496434: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.496237: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.496447: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.496460: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.496473: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.496532: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.497381: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.497738: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.497824: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.498268: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.498722: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.498829: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.498872: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.498886: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.498900: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.498914: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.498927: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.498941: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.499017: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.499197: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.499472: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.499507: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.499514: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.499520: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.499500: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.499549: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.499581: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.499592: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.499603: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.499614: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.499623: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.499662: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.499634: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.499687: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.499944: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.499979: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.499986: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.499992: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.500135: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.500541: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.500597: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.500632: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.500643: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.500654: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.500665: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.500675: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.500685: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.500759: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.500978: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.501035: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.501070: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.501081: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.501091: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.501102: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.501112: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.501123: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.501190: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.501327: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.501370: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.501811: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.502383: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.502911: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.503065: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.503100: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.503107: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.503114: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.503042: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:43.503263: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.503497: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:43.504165: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.504546: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.504578: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.504585: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.504592: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.504734: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.504945: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.505883: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.505925: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.505933: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.505957: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.506479: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.506607: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.506828: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.507613: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.507789: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.507829: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.507837: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.507843: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.508167: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.508372: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.508720: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.509421: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.509469: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.509501: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.509512: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.509522: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.509533: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.509542: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.509552: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.509611: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.509722: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.510114: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1b.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.510172: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.510204: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.510215: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.510225: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.510235: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.510245: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.510255: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.510328: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.511305: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.511464: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.511998: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.512934: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.512968: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.512975: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.512981: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.513119: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.513211: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.513351: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.513572: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.513641: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 5 2020-07-09 23:21:43.513675: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.513683: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 5 2020-07-09 23:21:43.513689: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 5: N 2020-07-09 23:21:43.513951: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.514028: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.514799: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.514932: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.515809: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.515861: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.515894: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.515905: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.515915: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.515926: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.515942: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.515953: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.516005: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.516183: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.516242: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.516274: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.516285: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.516296: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.516306: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.516317: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.516327: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.516399: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.516460: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.517279: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.517342: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.517374: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.517386: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.517396: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.517407: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.517417: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.517427: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.517504: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.517506: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.517700: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.518052: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.518188: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.518547: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.519347: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.519380: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.519387: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.519393: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.519533: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.519830: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.519862: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.519870: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.519875: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.519820: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.519868: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.519899: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.519909: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.519919: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.519929: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.519938: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.520015: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.519949: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.520007: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.520318: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.520373: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.520409: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.520421: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.520431: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.520442: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.520452: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.520462: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.520520: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.521055: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.521100: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 5, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1b.0, compute capability: 7.0) 2020-07-09 23:21:43.521200: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.521700: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.521697: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.522281: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.522703: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.522739: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.522746: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.522752: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.522908: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.522907: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:43.523353: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.523365: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.523395: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.523402: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.523408: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.523561: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.523921: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.523958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.523966: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.523973: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.524118: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.524558: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.524861: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.525233: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.525804: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.526881: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.527444: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.527733: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.527875: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.528126: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.528189: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.528232: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.528245: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.528258: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.528271: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.528283: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.528296: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.528370: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.529629: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.529683: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.529716: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.529727: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.529737: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.529747: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.529757: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.529768: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.529832: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.530076: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.531559: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.531710: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.531739: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.531745: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.531751: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.531668: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.531887: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.532240: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.533218: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.533249: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.533257: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.533263: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.533400: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.533421: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.533483: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.533525: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.533539: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.533552: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.533565: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.533577: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.533590: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.533647: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.535034: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.535064: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.535068: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.535556: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.535797: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.535867: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.535930: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.535947: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.535961: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.535975: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.535989: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.536002: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.536070: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.536723: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.536842: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.536909: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.536953: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.536968: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.536981: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.536994: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.537006: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.537020: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.537091: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.537199: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.537229: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.537236: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.537242: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.537384: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.537729: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.538810: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.538986: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.539261: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.539232: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.540483: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.540518: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.540526: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.540533: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.540687: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.540913: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.540942: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.540950: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.540955: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.541100: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.540958: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.540992: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.541046: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.541079: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.541090: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.541100: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.541110: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.541120: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.541131: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.541184: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.542380: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.542767: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.542917: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.544045: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:43.544392: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) 2020-07-09 23:21:43.544583: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.544616: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.544623: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.544629: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.544764: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.546450: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.548101: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.559088: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.560875: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:19.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.560934: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.560972: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.560986: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.561000: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.561013: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.561026: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.561039: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.561093: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.562796: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.564424: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 3 2020-07-09 23:21:43.564458: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.564466: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 3 2020-07-09 23:21:43.564473: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 3: N 2020-07-09 23:21:43.564610: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.566279: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.567939: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 3, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:19.0, compute capability: 7.0) 2020-07-09 23:21:43.576984: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.576982: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.578783: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.578845: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.578878: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.578890: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.578900: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.578911: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.578921: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.578932: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.579004: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.580410: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1d.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.580471: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.580504: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.580515: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.580526: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.580536: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.580545: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.580556: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.580630: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.580673: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.582317: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.582354: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.582361: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.582368: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.582514: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.584203: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.584285: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.585872: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.588165: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 7 2020-07-09 23:21:43.588226: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.588237: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 7 2020-07-09 23:21:43.588244: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 7: N 2020-07-09 23:21:43.588512: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.591065: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.592752: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 7, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1d.0, compute capability: 7.0) 2020-07-09 23:21:43.633437: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.635250: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:18.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.635316: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.635353: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.635365: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.635375: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.635386: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.635396: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.635407: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.635475: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.637151: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.638820: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 2 2020-07-09 23:21:43.638860: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.638868: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 2 2020-07-09 23:21:43.638874: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 2: N 2020-07-09 23:21:43.639028: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.640759: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.642412: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 2, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:18.0, compute capability: 7.0) 2020-07-09 23:21:43.666974: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.668776: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1a.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.668830: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.668865: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.668876: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.668886: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.668896: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.668906: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.668917: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.668986: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.670879: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.672517: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 4 2020-07-09 23:21:43.672552: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.672559: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 4 2020-07-09 23:21:43.672566: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 4: N 2020-07-09 23:21:43.672703: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.674548: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.677410: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 4, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1a.0, compute capability: 7.0) 2020-07-09 23:21:43.696779: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.700155: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:16.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.700209: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.700241: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.700251: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.700262: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.700272: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.700282: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.700293: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.700359: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.703960: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.707554: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 0 2020-07-09 23:21:43.707589: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.707597: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 0 2020-07-09 23:21:43.707603: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 0: N 2020-07-09 23:21:43.708471: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.711358: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.715057: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 0, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:16.0, compute capability: 7.0) 2020-07-09 23:21:43.846105: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.847939: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1555] Found device 0 with properties: pciBusID: 0000:00:1c.0 name: Tesla V100-SXM2-32GB computeCapability: 7.0 coreClock: 1.53GHz coreCount: 80 deviceMemorySize: 31.72GiB deviceMemoryBandwidth: 836.37GiB/s 2020-07-09 23:21:43.848002: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudart.so.10.1 2020-07-09 23:21:43.848037: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:43.848049: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcufft.so.10 2020-07-09 23:21:43.848059: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcurand.so.10 2020-07-09 23:21:43.848070: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusolver.so.10 2020-07-09 23:21:43.848080: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcusparse.so.10 2020-07-09 23:21:43.848092: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:43.848165: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.849870: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.851516: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1697] Adding visible gpu devices: 6 2020-07-09 23:21:43.851553: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1096] Device interconnect StreamExecutor with strength 1 edge matrix: 2020-07-09 23:21:43.851577: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1102] 6 2020-07-09 23:21:43.851584: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1115] 6: N 2020-07-09 23:21:43.851732: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.853404: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:981] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero 2020-07-09 23:21:43.855080: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1241] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 30525 MB memory) -> physical GPU (device: 6, name: Tesla V100-SXM2-32GB, pci bus id: 0000:00:1c.0, compute capability: 7.0) INFO:tensorflow:Running local_init_op. I0709 23:21:45.777247 139954201352000 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.809640 139728809436992 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.810378 139793959765824 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:45.880745 139954201352000 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.907049 140576162572096 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:45.913531 139728809436992 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.927654 140079197558592 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:45.929942 139793959765824 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.938606 140058100578112 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.941328 140347868399424 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.967229 140512030914368 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:45.983383 139719730001728 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.018986 140605867497280 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.021220 140111135848256 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.024233 140021899769664 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.026453 140576162572096 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.040025 140544370415424 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.047942 139648303425344 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.058110 140079197558592 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.058342 140347868399424 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.069336 140058100578112 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.077607 139738826934080 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.090559 140512030914368 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.096904 140101099308864 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.103105 139661611177792 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.106546 139719730001728 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.109820 140159716185920 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.115955 140496823912256 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.125174 139761894754112 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.128506 140345154484032 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.131434 140021899769664 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.146358 140111135848256 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.150194 140605867497280 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.154941 140691758417728 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.163142 139979895777088 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.173176 139648303425344 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.181858 139829151631168 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.182308 139982497134400 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.188552 140659616442176 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.189971 140339437655872 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.200254 139781685286720 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.205469 139738826934080 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.206223 140605020944192 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.212543 140422864979776 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.219002 140544370415424 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.221044 140033300801344 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.222849 140101099308864 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.223327 140154930235200 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.228314 139661611177792 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.230359 139834293708608 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.231639 140562346370880 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.234010 140159716185920 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.237833 140598316468032 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.240450 139898123876160 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.241483 139655422154560 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.241943 140496823912256 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.242932 139761894754112 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.252521 140439388542784 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.254439 140345154484032 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.262477 139767071725376 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.267130 139982119135040 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.276131 139979895777088 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.277865 140448709056320 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.279621 140493844432704 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.283060 140336345724736 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.284089 139806201362240 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.285412 140691758417728 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.286264 139684024837952 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.288404 140543473829696 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.291199 140458393626432 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.299165 140595026159424 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.300625 140571334534976 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.305359 139829151631168 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.309170 139958812215104 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.310109 139982497134400 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.314642 140659616442176 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.316066 140339437655872 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.324888 139781685286720 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.335175 140712519374656 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.336066 140605020944192 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.336326 140422864979776 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.338420 139889036683072 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.340414 139630126221120 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.343699 140033300801344 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.346833 140154930235200 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.348441 140136554874688 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.350919 139769147275072 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.357232 140562346370880 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.358969 139834293708608 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.359932 139853248595776 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.369523 140598316468032 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.370714 139655422154560 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.371811 139898123876160 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.377570 140439388542784 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.391108 139767071725376 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.400533 139982119135040 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.402174 140493844432704 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.403724 140448709056320 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.407160 140336345724736 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.411462 139684024837952 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.414928 139810623559488 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.415441 140511694276416 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.417023 140458393626432 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.418097 139795966580544 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.419832 139806201362240 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.423840 140543473829696 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.423758 140595026159424 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.425583 140571334534976 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.427023 140690824218432 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.434859 139958812215104 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.458461 139935576172352 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.460915 140712519374656 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.465666 139889036683072 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.466217 139630126221120 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.468196 139766263830336 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.475419 140136554874688 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.483666 139769147275072 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.488252 139853248595776 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.491563 140435199129408 session_manager.py:504] Running local_init_op. INFO:tensorflow:Running local_init_op. I0709 23:21:46.540114 140445836597056 session_manager.py:504] Running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.548413 139810623559488 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.548512 140511694276416 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.551860 139795966580544 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.560212 140690824218432 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.585011 139935576172352 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.596381 139766263830336 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.626021 140435199129408 session_manager.py:507] Done running local_init_op. INFO:tensorflow:Done running local_init_op. I0709 23:21:46.704327 140445836597056 session_manager.py:507] Done running local_init_op. Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up Running warm up 2020-07-09 23:21:52.037216: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.078468: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.228010: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.250824: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.280270: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.281580: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.285592: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.345508: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.347212: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.362317: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.365410: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.376565: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.387213: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.401314: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.415031: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.416087: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.418179: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.436831: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.447249: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.450316: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.451631: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.453266: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.456860: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.462980: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.466737: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.467360: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.476017: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.481515: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.487073: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.495241: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.495931: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.502229: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.510900: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.512733: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.519782: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.522022: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.522750: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.526025: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.525943: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.544370: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.544706: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.555786: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.556840: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.566870: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.567427: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.567979: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.594925: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.596964: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.597322: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.597941: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.602781: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.614867: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.620111: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.626728: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.632129: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.647968: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.656036: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.662042: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.664983: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.670185: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.671311: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.672135: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.682806: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.687473: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.697233: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.715620: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.723866: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.733560: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.738219: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.742962: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.747772: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.767427: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.768388: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.770715: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.784740: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.785189: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.809260: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.828426: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.829018: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.830373: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.832580: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.834707: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.837339: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.846398: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.849209: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.852683: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.854944: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.858162: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.867650: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.874252: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.883637: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.884040: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.888574: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.891196: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.896411: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.908580: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.915023: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.921565: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.924253: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.925418: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.928708: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.932166: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.938649: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.954457: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.960260: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:52.963979: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.965179: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.972848: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.985495: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:52.988769: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.002171: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.003169: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.005710: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.020867: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.024984: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:53.030610: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.032333: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.053412: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.080766: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.120544: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.122925: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:53.192800: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcublas.so.10 2020-07-09 23:21:53.237189: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.288297: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.295855: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.360812: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.463820: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 2020-07-09 23:21:53.535089: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library libcudnn.so.7 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Using network AWS Libfabric NCCL version 2.6.4+cuda10.1 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.167<0> tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.181<0> tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.142<0> tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.63<0> tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.188<0> tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.54.123<0> tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.43.174<0> tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.48.221<0> tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI Selected Provider is efa tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Using network AWS Libfabric tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Trees [0] 44/-1/-1->47->46|46->47->44/-1/-1 [1] 46/-1/-1->47->44|44->47->46/-1/-1 [2] 46/-1/-1->47->44|44->47->46/-1/-1 [3] 44/-1/-1->47->46|46->47->44/-1/-1 [4] 44/-1/-1->47->46|46->47->44/-1/-1 [5] 46/-1/-1->47->44|44->47->46/-1/-1 [6] 46/28/60->47->44|44->47->46/28/60 [7] 44/-1/-1->47->46|46->47->44/-1/-1 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Trees [0] 51/-1/-1->48->35|35->48->51/-1/-1 [1] 52/-1/-1->48->51|51->48->52/-1/-1 [2] -1/-1/-1->48->51|51->48->-1/-1/-1 [3] 51/-1/-1->48->52|52->48->51/-1/-1 [4] 51/-1/-1->48->59|59->48->51/-1/-1 [5] 52/-1/-1->48->51|51->48->52/-1/-1 [6] -1/-1/-1->48->51|51->48->-1/-1/-1 [7] 51/-1/-1->48->52|52->48->51/-1/-1 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Trees [0] 53/-1/-1->49->50|50->49->53/-1/-1 [1] 50/-1/-1->49->34|34->49->50/-1/-1 [2] 50/-1/-1->49->53|53->49->50/-1/-1 [3] -1/-1/-1->49->50|50->49->-1/-1/-1 [4] 53/-1/-1->49->50|50->49->53/-1/-1 [5] 50/-1/-1->49->58|58->49->50/-1/-1 [6] 50/-1/-1->49->53|53->49->50/-1/-1 [7] -1/-1/-1->49->50|50->49->-1/-1/-1 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 00 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Trees [0] 50/40/56->51->48|48->51->50/40/56 [1] 48/-1/-1->51->50|50->51->48/-1/-1 [2] 48/-1/-1->51->50|50->51->48/-1/-1 [3] 50/-1/-1->51->48|48->51->50/-1/-1 [4] 50/-1/-1->51->48|48->51->50/-1/-1 [5] 48/-1/-1->51->50|50->51->48/-1/-1 [6] 48/-1/-1->51->50|50->51->48/-1/-1 [7] 50/-1/-1->51->48|48->51->50/-1/-1 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Trees [0] 49/-1/-1->50->51|51->50->49/-1/-1 [1] 51/41/57->50->49|49->50->51/41/57 [2] 51/-1/-1->50->49|49->50->51/-1/-1 [3] 49/-1/-1->50->51|51->50->49/-1/-1 [4] 49/-1/-1->50->51|51->50->49/-1/-1 [5] 51/-1/-1->50->49|49->50->51/-1/-1 [6] 51/-1/-1->50->49|49->50->51/-1/-1 [7] 49/-1/-1->50->51|51->50->49/-1/-1 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Trees [0] 52/-1/-1->55->54|54->55->52/-1/-1 [1] 54/-1/-1->55->52|52->55->54/-1/-1 [2] 54/44/60->55->52|52->55->54/44/60 [3] 52/-1/-1->55->54|54->55->52/-1/-1 [4] 52/-1/-1->55->54|54->55->52/-1/-1 [5] 54/-1/-1->55->52|52->55->54/-1/-1 [6] 54/-1/-1->55->52|52->55->54/-1/-1 [7] 52/-1/-1->55->54|54->55->52/-1/-1 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Trees [0] 54/-1/-1->53->49|49->53->54/-1/-1 [1] -1/-1/-1->53->54|54->53->-1/-1/-1 [2] 49/-1/-1->53->54|54->53->49/-1/-1 [3] 54/-1/-1->53->38|38->53->54/-1/-1 [4] 54/-1/-1->53->49|49->53->54/-1/-1 [5] -1/-1/-1->53->54|54->53->-1/-1/-1 [6] 49/-1/-1->53->54|54->53->49/-1/-1 [7] 54/-1/-1->53->62|62->53->54/-1/-1 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Trees [0] 55/-1/-1->54->53|53->54->55/-1/-1 [1] 53/-1/-1->54->55|55->54->53/-1/-1 [2] 53/-1/-1->54->55|55->54->53/-1/-1 [3] 55/45/61->54->53|53->54->55/45/61 [4] 55/-1/-1->54->53|53->54->55/-1/-1 [5] 53/-1/-1->54->55|55->54->53/-1/-1 [6] 53/-1/-1->54->55|55->54->53/-1/-1 [7] 55/-1/-1->54->53|53->54->55/-1/-1 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Trees [0] -1/-1/-1->52->55|55->52->-1/-1/-1 [1] 55/-1/-1->52->48|48->52->55/-1/-1 [2] 55/-1/-1->52->39|39->52->55/-1/-1 [3] 48/-1/-1->52->55|55->52->48/-1/-1 [4] -1/-1/-1->52->55|55->52->-1/-1/-1 [5] 55/-1/-1->52->48|48->52->55/-1/-1 [6] 55/-1/-1->52->63|63->52->55/-1/-1 [7] 48/-1/-1->52->55|55->52->48/-1/-1 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Trees [0] 57/-1/-1->58->59|59->58->57/-1/-1 [1] 59/-1/-1->58->57|57->58->59/-1/-1 [2] 59/-1/-1->58->57|57->58->59/-1/-1 [3] 57/-1/-1->58->59|59->58->57/-1/-1 [4] 57/-1/-1->58->59|59->58->57/-1/-1 [5] 59/49/1->58->57|57->58->59/49/1 [6] 59/-1/-1->58->57|57->58->59/-1/-1 [7] 57/-1/-1->58->59|59->58->57/-1/-1 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Trees [0] 59/-1/-1->56->51|51->56->59/-1/-1 [1] 60/-1/-1->56->59|59->56->60/-1/-1 [2] -1/-1/-1->56->59|59->56->-1/-1/-1 [3] 59/-1/-1->56->60|60->56->59/-1/-1 [4] 59/-1/-1->56->43|43->56->59/-1/-1 [5] 60/-1/-1->56->59|59->56->60/-1/-1 [6] -1/-1/-1->56->59|59->56->-1/-1/-1 [7] 59/-1/-1->56->60|60->56->59/-1/-1 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Trees [0] 60/-1/-1->63->62|62->63->60/-1/-1 [1] 62/-1/-1->63->60|60->63->62/-1/-1 [2] 62/-1/-1->63->60|60->63->62/-1/-1 [3] 60/-1/-1->63->62|62->63->60/-1/-1 [4] 60/-1/-1->63->62|62->63->60/-1/-1 [5] 62/-1/-1->63->60|60->63->62/-1/-1 [6] 62/52/4->63->60|60->63->62/52/4 [7] 60/-1/-1->63->62|62->63->60/-1/-1 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Trees [0] 62/-1/-1->61->57|57->61->62/-1/-1 [1] -1/-1/-1->61->62|62->61->-1/-1/-1 [2] 57/-1/-1->61->62|62->61->57/-1/-1 [3] 62/-1/-1->61->54|54->61->62/-1/-1 [4] 62/-1/-1->61->57|57->61->62/-1/-1 [5] -1/-1/-1->61->62|62->61->-1/-1/-1 [6] 57/-1/-1->61->62|62->61->57/-1/-1 [7] 62/-1/-1->61->46|46->61->62/-1/-1 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Trees [0] 61/-1/-1->57->58|58->57->61/-1/-1 [1] 58/-1/-1->57->50|50->57->58/-1/-1 [2] 58/-1/-1->57->61|61->57->58/-1/-1 [3] -1/-1/-1->57->58|58->57->-1/-1/-1 [4] 61/-1/-1->57->58|58->57->61/-1/-1 [5] 58/-1/-1->57->42|42->57->58/-1/-1 [6] 58/-1/-1->57->61|61->57->58/-1/-1 [7] -1/-1/-1->57->58|58->57->-1/-1/-1 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Trees [0] -1/-1/-1->60->63|63->60->-1/-1/-1 [1] 63/-1/-1->60->56|56->60->63/-1/-1 [2] 63/-1/-1->60->55|55->60->63/-1/-1 [3] 56/-1/-1->60->63|63->60->56/-1/-1 [4] -1/-1/-1->60->63|63->60->-1/-1/-1 [5] 63/-1/-1->60->56|56->60->63/-1/-1 [6] 63/-1/-1->60->47|47->60->63/-1/-1 [7] 56/-1/-1->60->63|63->60->56/-1/-1 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Trees [0] 58/-1/-1->59->56|56->59->58/-1/-1 [1] 56/-1/-1->59->58|58->59->56/-1/-1 [2] 56/-1/-1->59->58|58->59->56/-1/-1 [3] 58/-1/-1->59->56|56->59->58/-1/-1 [4] 58/48/0->59->56|56->59->58/48/0 [5] 56/-1/-1->59->58|58->59->56/-1/-1 [6] 56/-1/-1->59->58|58->59->56/-1/-1 [7] 58/-1/-1->59->56|56->59->58/-1/-1 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Trees [0] 63/-1/-1->62->61|61->62->63/-1/-1 [1] 61/-1/-1->62->63|63->62->61/-1/-1 [2] 61/-1/-1->62->63|63->62->61/-1/-1 [3] 63/-1/-1->62->61|61->62->63/-1/-1 [4] 63/-1/-1->62->61|61->62->63/-1/-1 [5] 61/-1/-1->62->63|63->62->61/-1/-1 [6] 61/-1/-1->62->63|63->62->61/-1/-1 [7] 63/53/5->62->61|61->62->63/53/5 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Trees [0] 1/-1/-1->2->3|3->2->1/-1/-1 [1] 3/33/-1->2->1|1->2->3/33/-1 [2] 3/-1/-1->2->1|1->2->3/-1/-1 [3] 1/-1/-1->2->3|3->2->1/-1/-1 [4] 1/-1/-1->2->3|3->2->1/-1/-1 [5] 3/-1/-1->2->1|1->2->3/-1/-1 [6] 3/-1/-1->2->1|1->2->3/-1/-1 [7] 1/-1/-1->2->3|3->2->1/-1/-1 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Trees [0] 5/-1/-1->1->2|2->1->5/-1/-1 [1] 2/-1/-1->1->-1|-1->1->2/-1/-1 [2] 2/-1/-1->1->5|5->1->2/-1/-1 [3] -1/-1/-1->1->2|2->1->-1/-1/-1 [4] 5/-1/-1->1->2|2->1->5/-1/-1 [5] 2/-1/-1->1->58|58->1->2/-1/-1 [6] 2/-1/-1->1->5|5->1->2/-1/-1 [7] -1/-1/-1->1->2|2->1->-1/-1/-1 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Trees [0] 2/32/-1->3->0|0->3->2/32/-1 [1] 0/-1/-1->3->2|2->3->0/-1/-1 [2] 0/-1/-1->3->2|2->3->0/-1/-1 [3] 2/-1/-1->3->0|0->3->2/-1/-1 [4] 2/-1/-1->3->0|0->3->2/-1/-1 [5] 0/-1/-1->3->2|2->3->0/-1/-1 [6] 0/-1/-1->3->2|2->3->0/-1/-1 [7] 2/-1/-1->3->0|0->3->2/-1/-1 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 00/08 : 0 3 2 1 5 6 7 4 8 11 10 9 13 14 15 12 16 19 18 17 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 01/08 : 0 4 7 6 5 9 10 11 8 12 15 14 13 17 18 19 16 20 23 22 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 02/08 : 0 1 5 6 10 11 15 12 8 9 13 14 18 19 23 20 16 17 21 22 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 03/08 : 0 1 2 6 5 4 7 11 8 9 10 14 13 12 15 19 16 17 18 22 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 04/08 : 0 3 2 1 5 6 7 4 8 11 10 9 13 14 15 12 16 19 18 17 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 05/08 : 0 4 7 6 5 9 10 11 8 12 15 14 13 17 18 19 16 20 23 22 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 06/08 : 0 1 5 6 10 11 15 12 8 9 13 14 18 19 23 20 16 17 21 22 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Channel 07/08 : 0 1 2 6 5 4 7 11 8 9 10 14 13 12 15 19 16 17 18 22 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Trees [0] 3/-1/-1->0->-1|-1->0->3/-1/-1 [1] 4/-1/-1->0->3|3->0->4/-1/-1 [2] -1/-1/-1->0->3|3->0->-1/-1/-1 [3] 3/-1/-1->0->4|4->0->3/-1/-1 [4] 3/-1/-1->0->59|59->0->3/-1/-1 [5] 4/-1/-1->0->3|3->0->4/-1/-1 [6] -1/-1/-1->0->3|3->0->-1/-1/-1 [7] 3/-1/-1->0->4|4->0->3/-1/-1 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Trees [0] -1/-1/-1->4->7|7->4->-1/-1/-1 [1] 7/-1/-1->4->0|0->4->7/-1/-1 [2] 7/-1/-1->4->-1|-1->4->7/-1/-1 [3] 0/-1/-1->4->7|7->4->0/-1/-1 [4] -1/-1/-1->4->7|7->4->-1/-1/-1 [5] 7/-1/-1->4->0|0->4->7/-1/-1 [6] 7/-1/-1->4->63|63->4->7/-1/-1 [7] 0/-1/-1->4->7|7->4->0/-1/-1 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Trees [0] 6/-1/-1->5->1|1->5->6/-1/-1 [1] -1/-1/-1->5->6|6->5->-1/-1/-1 [2] 1/-1/-1->5->6|6->5->1/-1/-1 [3] 6/-1/-1->5->-1|-1->5->6/-1/-1 [4] 6/-1/-1->5->1|1->5->6/-1/-1 [5] -1/-1/-1->5->6|6->5->-1/-1/-1 [6] 1/-1/-1->5->6|6->5->1/-1/-1 [7] 6/-1/-1->5->62|62->5->6/-1/-1 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Trees [0] 7/-1/-1->6->5|5->6->7/-1/-1 [1] 5/-1/-1->6->7|7->6->5/-1/-1 [2] 5/-1/-1->6->7|7->6->5/-1/-1 [3] 7/37/-1->6->5|5->6->7/37/-1 [4] 7/-1/-1->6->5|5->6->7/-1/-1 [5] 5/-1/-1->6->7|7->6->5/-1/-1 [6] 5/-1/-1->6->7|7->6->5/-1/-1 [7] 7/-1/-1->6->5|5->6->7/-1/-1 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Trees [0] 4/-1/-1->7->6|6->7->4/-1/-1 [1] 6/-1/-1->7->4|4->7->6/-1/-1 [2] 6/36/-1->7->4|4->7->6/36/-1 [3] 4/-1/-1->7->6|6->7->4/-1/-1 [4] 4/-1/-1->7->6|6->7->4/-1/-1 [5] 6/-1/-1->7->4|4->7->6/-1/-1 [6] 6/-1/-1->7->4|4->7->6/-1/-1 [7] 4/-1/-1->7->6|6->7->4/-1/-1 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Trees [0] 13/-1/-1->9->10|10->9->13/-1/-1 [1] 10/-1/-1->9->18|18->9->10/-1/-1 [2] 10/-1/-1->9->13|13->9->10/-1/-1 [3] -1/-1/-1->9->10|10->9->-1/-1/-1 [4] 13/-1/-1->9->10|10->9->13/-1/-1 [5] 10/-1/-1->9->-1|-1->9->10/-1/-1 [6] 10/-1/-1->9->13|13->9->10/-1/-1 [7] -1/-1/-1->9->10|10->9->-1/-1/-1 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Trees [0] -1/-1/-1->12->15|15->12->-1/-1/-1 [1] 15/-1/-1->12->8|8->12->15/-1/-1 [2] 15/-1/-1->12->23|23->12->15/-1/-1 [3] 8/-1/-1->12->15|15->12->8/-1/-1 [4] -1/-1/-1->12->15|15->12->-1/-1/-1 [5] 15/-1/-1->12->8|8->12->15/-1/-1 [6] 15/-1/-1->12->-1|-1->12->15/-1/-1 [7] 8/-1/-1->12->15|15->12->8/-1/-1 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Trees [0] 9/-1/-1->10->11|11->10->9/-1/-1 [1] 11/-1/-1->10->9|9->10->11/-1/-1 [2] 11/-1/-1->10->9|9->10->11/-1/-1 [3] 9/-1/-1->10->11|11->10->9/-1/-1 [4] 9/-1/-1->10->11|11->10->9/-1/-1 [5] 11/41/-1->10->9|9->10->11/41/-1 [6] 11/-1/-1->10->9|9->10->11/-1/-1 [7] 9/-1/-1->10->11|11->10->9/-1/-1 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Trees [0] 10/-1/-1->11->8|8->11->10/-1/-1 [1] 8/-1/-1->11->10|10->11->8/-1/-1 [2] 8/-1/-1->11->10|10->11->8/-1/-1 [3] 10/-1/-1->11->8|8->11->10/-1/-1 [4] 10/40/-1->11->8|8->11->10/40/-1 [5] 8/-1/-1->11->10|10->11->8/-1/-1 [6] 8/-1/-1->11->10|10->11->8/-1/-1 [7] 10/-1/-1->11->8|8->11->10/-1/-1 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Trees [0] 11/-1/-1->8->19|19->8->11/-1/-1 [1] 12/-1/-1->8->11|11->8->12/-1/-1 [2] -1/-1/-1->8->11|11->8->-1/-1/-1 [3] 11/-1/-1->8->12|12->8->11/-1/-1 [4] 11/-1/-1->8->-1|-1->8->11/-1/-1 [5] 12/-1/-1->8->11|11->8->12/-1/-1 [6] -1/-1/-1->8->11|11->8->-1/-1/-1 [7] 11/-1/-1->8->12|12->8->11/-1/-1 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Trees [0] 14/-1/-1->13->9|9->13->14/-1/-1 [1] -1/-1/-1->13->14|14->13->-1/-1/-1 [2] 9/-1/-1->13->14|14->13->9/-1/-1 [3] 14/-1/-1->13->22|22->13->14/-1/-1 [4] 14/-1/-1->13->9|9->13->14/-1/-1 [5] -1/-1/-1->13->14|14->13->-1/-1/-1 [6] 9/-1/-1->13->14|14->13->9/-1/-1 [7] 14/-1/-1->13->-1|-1->13->14/-1/-1 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Trees [0] 12/-1/-1->15->14|14->15->12/-1/-1 [1] 14/-1/-1->15->12|12->15->14/-1/-1 [2] 14/-1/-1->15->12|12->15->14/-1/-1 [3] 12/-1/-1->15->14|14->15->12/-1/-1 [4] 12/-1/-1->15->14|14->15->12/-1/-1 [5] 14/-1/-1->15->12|12->15->14/-1/-1 [6] 14/44/-1->15->12|12->15->14/44/-1 [7] 12/-1/-1->15->14|14->15->12/-1/-1 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Trees [0] 15/-1/-1->14->13|13->14->15/-1/-1 [1] 13/-1/-1->14->15|15->14->13/-1/-1 [2] 13/-1/-1->14->15|15->14->13/-1/-1 [3] 15/-1/-1->14->13|13->14->15/-1/-1 [4] 15/-1/-1->14->13|13->14->15/-1/-1 [5] 13/-1/-1->14->15|15->14->13/-1/-1 [6] 13/-1/-1->14->15|15->14->13/-1/-1 [7] 15/45/-1->14->13|13->14->15/45/-1 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Trees [0] 19/-1/-1->16->35|35->16->19/-1/-1 [1] 20/-1/-1->16->19|19->16->20/-1/-1 [2] -1/-1/-1->16->19|19->16->-1/-1/-1 [3] 19/-1/-1->16->20|20->16->19/-1/-1 [4] 19/-1/-1->16->27|27->16->19/-1/-1 [5] 20/-1/-1->16->19|19->16->20/-1/-1 [6] -1/-1/-1->16->19|19->16->-1/-1/-1 [7] 19/-1/-1->16->20|20->16->19/-1/-1 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Trees [0] 21/-1/-1->17->18|18->17->21/-1/-1 [1] 18/-1/-1->17->34|34->17->18/-1/-1 [2] 18/-1/-1->17->21|21->17->18/-1/-1 [3] -1/-1/-1->17->18|18->17->-1/-1/-1 [4] 21/-1/-1->17->18|18->17->21/-1/-1 [5] 18/-1/-1->17->26|26->17->18/-1/-1 [6] 18/-1/-1->17->21|21->17->18/-1/-1 [7] -1/-1/-1->17->18|18->17->-1/-1/-1 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Trees [0] 17/-1/-1->18->19|19->18->17/-1/-1 [1] 19/9/25->18->17|17->18->19/9/25 [2] 19/-1/-1->18->17|17->18->19/-1/-1 [3] 17/-1/-1->18->19|19->18->17/-1/-1 [4] 17/-1/-1->18->19|19->18->17/-1/-1 [5] 19/-1/-1->18->17|17->18->19/-1/-1 [6] 19/-1/-1->18->17|17->18->19/-1/-1 [7] 17/-1/-1->18->19|19->18->17/-1/-1 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Trees [0] 22/-1/-1->21->17|17->21->22/-1/-1 [1] -1/-1/-1->21->22|22->21->-1/-1/-1 [2] 17/-1/-1->21->22|22->21->17/-1/-1 [3] 22/-1/-1->21->38|38->21->22/-1/-1 [4] 22/-1/-1->21->17|17->21->22/-1/-1 [5] -1/-1/-1->21->22|22->21->-1/-1/-1 [6] 17/-1/-1->21->22|22->21->17/-1/-1 [7] 22/-1/-1->21->30|30->21->22/-1/-1 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Trees [0] 20/-1/-1->23->22|22->23->20/-1/-1 [1] 22/-1/-1->23->20|20->23->22/-1/-1 [2] 22/12/28->23->20|20->23->22/12/28 [3] 20/-1/-1->23->22|22->23->20/-1/-1 [4] 20/-1/-1->23->22|22->23->20/-1/-1 [5] 22/-1/-1->23->20|20->23->22/-1/-1 [6] 22/-1/-1->23->20|20->23->22/-1/-1 [7] 20/-1/-1->23->22|22->23->20/-1/-1 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Trees [0] 18/8/24->19->16|16->19->18/8/24 [1] 16/-1/-1->19->18|18->19->16/-1/-1 [2] 16/-1/-1->19->18|18->19->16/-1/-1 [3] 18/-1/-1->19->16|16->19->18/-1/-1 [4] 18/-1/-1->19->16|16->19->18/-1/-1 [5] 16/-1/-1->19->18|18->19->16/-1/-1 [6] 16/-1/-1->19->18|18->19->16/-1/-1 [7] 18/-1/-1->19->16|16->19->18/-1/-1 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Trees [0] -1/-1/-1->20->23|23->20->-1/-1/-1 [1] 23/-1/-1->20->16|16->20->23/-1/-1 [2] 23/-1/-1->20->39|39->20->23/-1/-1 [3] 16/-1/-1->20->23|23->20->16/-1/-1 [4] -1/-1/-1->20->23|23->20->-1/-1/-1 [5] 23/-1/-1->20->16|16->20->23/-1/-1 [6] 23/-1/-1->20->31|31->20->23/-1/-1 [7] 16/-1/-1->20->23|23->20->16/-1/-1 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Trees [0] 23/-1/-1->22->21|21->22->23/-1/-1 [1] 21/-1/-1->22->23|23->22->21/-1/-1 [2] 21/-1/-1->22->23|23->22->21/-1/-1 [3] 23/13/29->22->21|21->22->23/13/29 [4] 23/-1/-1->22->21|21->22->23/-1/-1 [5] 21/-1/-1->22->23|23->22->21/-1/-1 [6] 21/-1/-1->22->23|23->22->21/-1/-1 [7] 23/-1/-1->22->21|21->22->23/-1/-1 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Trees [0] 27/-1/-1->24->19|19->24->27/-1/-1 [1] 28/-1/-1->24->27|27->24->28/-1/-1 [2] -1/-1/-1->24->27|27->24->-1/-1/-1 [3] 27/-1/-1->24->28|28->24->27/-1/-1 [4] 27/-1/-1->24->43|43->24->27/-1/-1 [5] 28/-1/-1->24->27|27->24->28/-1/-1 [6] -1/-1/-1->24->27|27->24->-1/-1/-1 [7] 27/-1/-1->24->28|28->24->27/-1/-1 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Trees [0] 29/-1/-1->25->26|26->25->29/-1/-1 [1] 26/-1/-1->25->18|18->25->26/-1/-1 [2] 26/-1/-1->25->29|29->25->26/-1/-1 [3] -1/-1/-1->25->26|26->25->-1/-1/-1 [4] 29/-1/-1->25->26|26->25->29/-1/-1 [5] 26/-1/-1->25->42|42->25->26/-1/-1 [6] 26/-1/-1->25->29|29->25->26/-1/-1 [7] -1/-1/-1->25->26|26->25->-1/-1/-1 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Trees [0] 25/-1/-1->26->27|27->26->25/-1/-1 [1] 27/-1/-1->26->25|25->26->27/-1/-1 [2] 27/-1/-1->26->25|25->26->27/-1/-1 [3] 25/-1/-1->26->27|27->26->25/-1/-1 [4] 25/-1/-1->26->27|27->26->25/-1/-1 [5] 27/17/33->26->25|25->26->27/17/33 [6] 27/-1/-1->26->25|25->26->27/-1/-1 [7] 25/-1/-1->26->27|27->26->25/-1/-1 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Trees [0] 26/-1/-1->27->24|24->27->26/-1/-1 [1] 24/-1/-1->27->26|26->27->24/-1/-1 [2] 24/-1/-1->27->26|26->27->24/-1/-1 [3] 26/-1/-1->27->24|24->27->26/-1/-1 [4] 26/16/32->27->24|24->27->26/16/32 [5] 24/-1/-1->27->26|26->27->24/-1/-1 [6] 24/-1/-1->27->26|26->27->24/-1/-1 [7] 26/-1/-1->27->24|24->27->26/-1/-1 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Trees [0] -1/-1/-1->28->31|31->28->-1/-1/-1 [1] 31/-1/-1->28->24|24->28->31/-1/-1 [2] 31/-1/-1->28->23|23->28->31/-1/-1 [3] 24/-1/-1->28->31|31->28->24/-1/-1 [4] -1/-1/-1->28->31|31->28->-1/-1/-1 [5] 31/-1/-1->28->24|24->28->31/-1/-1 [6] 31/-1/-1->28->47|47->28->31/-1/-1 [7] 24/-1/-1->28->31|31->28->24/-1/-1 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Trees [0] 30/-1/-1->29->25|25->29->30/-1/-1 [1] -1/-1/-1->29->30|30->29->-1/-1/-1 [2] 25/-1/-1->29->30|30->29->25/-1/-1 [3] 30/-1/-1->29->22|22->29->30/-1/-1 [4] 30/-1/-1->29->25|25->29->30/-1/-1 [5] -1/-1/-1->29->30|30->29->-1/-1/-1 [6] 25/-1/-1->29->30|30->29->25/-1/-1 [7] 30/-1/-1->29->46|46->29->30/-1/-1 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Trees [0] 31/-1/-1->30->29|29->30->31/-1/-1 [1] 29/-1/-1->30->31|31->30->29/-1/-1 [2] 29/-1/-1->30->31|31->30->29/-1/-1 [3] 31/-1/-1->30->29|29->30->31/-1/-1 [4] 31/-1/-1->30->29|29->30->31/-1/-1 [5] 29/-1/-1->30->31|31->30->29/-1/-1 [6] 29/-1/-1->30->31|31->30->29/-1/-1 [7] 31/21/37->30->29|29->30->31/21/37 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Trees [0] 28/-1/-1->31->30|30->31->28/-1/-1 [1] 30/-1/-1->31->28|28->31->30/-1/-1 [2] 30/-1/-1->31->28|28->31->30/-1/-1 [3] 28/-1/-1->31->30|30->31->28/-1/-1 [4] 28/-1/-1->31->30|30->31->28/-1/-1 [5] 30/-1/-1->31->28|28->31->30/-1/-1 [6] 30/20/36->31->28|28->31->30/20/36 [7] 28/-1/-1->31->30|30->31->28/-1/-1 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Trees [0] 33/-1/-1->34->35|35->34->33/-1/-1 [1] 35/17/49->34->33|33->34->35/17/49 [2] 35/-1/-1->34->33|33->34->35/-1/-1 [3] 33/-1/-1->34->35|35->34->33/-1/-1 [4] 33/-1/-1->34->35|35->34->33/-1/-1 [5] 35/-1/-1->34->33|33->34->35/-1/-1 [6] 35/-1/-1->34->33|33->34->35/-1/-1 [7] 33/-1/-1->34->35|35->34->33/-1/-1 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Trees [0] 35/-1/-1->32->3|3->32->35/-1/-1 [1] 36/-1/-1->32->35|35->32->36/-1/-1 [2] -1/-1/-1->32->35|35->32->-1/-1/-1 [3] 35/-1/-1->32->36|36->32->35/-1/-1 [4] 35/-1/-1->32->27|27->32->35/-1/-1 [5] 36/-1/-1->32->35|35->32->36/-1/-1 [6] -1/-1/-1->32->35|35->32->-1/-1/-1 [7] 35/-1/-1->32->36|36->32->35/-1/-1 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Trees [0] 34/16/48->35->32|32->35->34/16/48 [1] 32/-1/-1->35->34|34->35->32/-1/-1 [2] 32/-1/-1->35->34|34->35->32/-1/-1 [3] 34/-1/-1->35->32|32->35->34/-1/-1 [4] 34/-1/-1->35->32|32->35->34/-1/-1 [5] 32/-1/-1->35->34|34->35->32/-1/-1 [6] 32/-1/-1->35->34|34->35->32/-1/-1 [7] 34/-1/-1->35->32|32->35->34/-1/-1 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Trees [0] 37/-1/-1->33->34|34->33->37/-1/-1 [1] 34/-1/-1->33->2|2->33->34/-1/-1 [2] 34/-1/-1->33->37|37->33->34/-1/-1 [3] -1/-1/-1->33->34|34->33->-1/-1/-1 [4] 37/-1/-1->33->34|34->33->37/-1/-1 [5] 34/-1/-1->33->26|26->33->34/-1/-1 [6] 34/-1/-1->33->37|37->33->34/-1/-1 [7] -1/-1/-1->33->34|34->33->-1/-1/-1 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Trees [0] -1/-1/-1->36->39|39->36->-1/-1/-1 [1] 39/-1/-1->36->32|32->36->39/-1/-1 [2] 39/-1/-1->36->7|7->36->39/-1/-1 [3] 32/-1/-1->36->39|39->36->32/-1/-1 [4] -1/-1/-1->36->39|39->36->-1/-1/-1 [5] 39/-1/-1->36->32|32->36->39/-1/-1 [6] 39/-1/-1->36->31|31->36->39/-1/-1 [7] 32/-1/-1->36->39|39->36->32/-1/-1 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Trees [0] 38/-1/-1->37->33|33->37->38/-1/-1 [1] -1/-1/-1->37->38|38->37->-1/-1/-1 [2] 33/-1/-1->37->38|38->37->33/-1/-1 [3] 38/-1/-1->37->6|6->37->38/-1/-1 [4] 38/-1/-1->37->33|33->37->38/-1/-1 [5] -1/-1/-1->37->38|38->37->-1/-1/-1 [6] 33/-1/-1->37->38|38->37->33/-1/-1 [7] 38/-1/-1->37->30|30->37->38/-1/-1 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Trees [0] 39/-1/-1->38->37|37->38->39/-1/-1 [1] 37/-1/-1->38->39|39->38->37/-1/-1 [2] 37/-1/-1->38->39|39->38->37/-1/-1 [3] 39/21/53->38->37|37->38->39/21/53 [4] 39/-1/-1->38->37|37->38->39/-1/-1 [5] 37/-1/-1->38->39|39->38->37/-1/-1 [6] 37/-1/-1->38->39|39->38->37/-1/-1 [7] 39/-1/-1->38->37|37->38->39/-1/-1 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Trees [0] 36/-1/-1->39->38|38->39->36/-1/-1 [1] 38/-1/-1->39->36|36->39->38/-1/-1 [2] 38/20/52->39->36|36->39->38/20/52 [3] 36/-1/-1->39->38|38->39->36/-1/-1 [4] 36/-1/-1->39->38|38->39->36/-1/-1 [5] 38/-1/-1->39->36|36->39->38/-1/-1 [6] 38/-1/-1->39->36|36->39->38/-1/-1 [7] 36/-1/-1->39->38|38->39->36/-1/-1 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Trees [0] 41/-1/-1->42->43|43->42->41/-1/-1 [1] 43/-1/-1->42->41|41->42->43/-1/-1 [2] 43/-1/-1->42->41|41->42->43/-1/-1 [3] 41/-1/-1->42->43|43->42->41/-1/-1 [4] 41/-1/-1->42->43|43->42->41/-1/-1 [5] 43/25/57->42->41|41->42->43/25/57 [6] 43/-1/-1->42->41|41->42->43/-1/-1 [7] 41/-1/-1->42->43|43->42->41/-1/-1 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Trees [0] 43/-1/-1->40->51|51->40->43/-1/-1 [1] 44/-1/-1->40->43|43->40->44/-1/-1 [2] -1/-1/-1->40->43|43->40->-1/-1/-1 [3] 43/-1/-1->40->44|44->40->43/-1/-1 [4] 43/-1/-1->40->11|11->40->43/-1/-1 [5] 44/-1/-1->40->43|43->40->44/-1/-1 [6] -1/-1/-1->40->43|43->40->-1/-1/-1 [7] 43/-1/-1->40->44|44->40->43/-1/-1 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Trees [0] 42/-1/-1->43->40|40->43->42/-1/-1 [1] 40/-1/-1->43->42|42->43->40/-1/-1 [2] 40/-1/-1->43->42|42->43->40/-1/-1 [3] 42/-1/-1->43->40|40->43->42/-1/-1 [4] 42/24/56->43->40|40->43->42/24/56 [5] 40/-1/-1->43->42|42->43->40/-1/-1 [6] 40/-1/-1->43->42|42->43->40/-1/-1 [7] 42/-1/-1->43->40|40->43->42/-1/-1 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Trees [0] 45/-1/-1->41->42|42->41->45/-1/-1 [1] 42/-1/-1->41->50|50->41->42/-1/-1 [2] 42/-1/-1->41->45|45->41->42/-1/-1 [3] -1/-1/-1->41->42|42->41->-1/-1/-1 [4] 45/-1/-1->41->42|42->41->45/-1/-1 [5] 42/-1/-1->41->10|10->41->42/-1/-1 [6] 42/-1/-1->41->45|45->41->42/-1/-1 [7] -1/-1/-1->41->42|42->41->-1/-1/-1 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Trees [0] -1/-1/-1->44->47|47->44->-1/-1/-1 [1] 47/-1/-1->44->40|40->44->47/-1/-1 [2] 47/-1/-1->44->55|55->44->47/-1/-1 [3] 40/-1/-1->44->47|47->44->40/-1/-1 [4] -1/-1/-1->44->47|47->44->-1/-1/-1 [5] 47/-1/-1->44->40|40->44->47/-1/-1 [6] 47/-1/-1->44->15|15->44->47/-1/-1 [7] 40/-1/-1->44->47|47->44->40/-1/-1 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Trees [0] 46/-1/-1->45->41|41->45->46/-1/-1 [1] -1/-1/-1->45->46|46->45->-1/-1/-1 [2] 41/-1/-1->45->46|46->45->41/-1/-1 [3] 46/-1/-1->45->54|54->45->46/-1/-1 [4] 46/-1/-1->45->41|41->45->46/-1/-1 [5] -1/-1/-1->45->46|46->45->-1/-1/-1 [6] 41/-1/-1->45->46|46->45->41/-1/-1 [7] 46/-1/-1->45->14|14->45->46/-1/-1 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Trees [0] 47/-1/-1->46->45|45->46->47/-1/-1 [1] 45/-1/-1->46->47|47->46->45/-1/-1 [2] 45/-1/-1->46->47|47->46->45/-1/-1 [3] 47/-1/-1->46->45|45->46->47/-1/-1 [4] 47/-1/-1->46->45|45->46->47/-1/-1 [5] 45/-1/-1->46->47|47->46->45/-1/-1 [6] 45/-1/-1->46->47|47->46->45/-1/-1 [7] 47/29/61->46->45|45->46->47/29/61 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 00 : 49[170] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 00 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 00 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 00 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 00 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 00 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 00 : 44[1a0] -> 48[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 00 : 1[170] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 00 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 00 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 00 : 57[170] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 00 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 00 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 00 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 00 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 00 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 00 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 00 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 00 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 00 : 52[1a0] -> 56[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 00 : 9[170] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 00 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 00 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 00 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 00 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 00 : 60[1a0] -> 0[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 00 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 00 : 25[170] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 00 : 4[1a0] -> 8[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 00 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 00 : 60[1a0] -> 0[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 00 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 00 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 00 : 17[170] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 00 : 33[170] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 00 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 00 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 00 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 00 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 00 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 00 : 20[1a0] -> 24[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 00 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 00 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 00 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 00 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 00 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 00 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 00 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 00 : 41[170] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 00 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 00 : 52[1a0] -> 56[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 00 : 12[1a0] -> 16[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 00 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 00 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 00 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 00 : 28[1a0] -> 32[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 00 : 12[1a0] -> 16[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 00 : 4[1a0] -> 8[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 00 : 36[1a0] -> 40[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 00 : 28[1a0] -> 32[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 00 : 36[1a0] -> 40[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 00 : 20[1a0] -> 24[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 00 : 44[1a0] -> 48[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 00 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 00 : 53[1b0] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 00 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 00 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 00 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 00 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 00 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 00 : 61[1b0] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 00 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 00 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 00 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 00 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 00 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 00 : 13[1b0] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 00 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 00 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 00 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 00 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 00 : 29[1b0] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 00 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 00 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 00 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 00 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 01 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 00 : 45[1b0] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 00 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 00 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 00 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 00 : 37[1b0] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 00 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 00 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 01 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 00 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 00 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 00 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 00 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 00 : 21[1b0] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 01 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 00 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 00 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 00 : 5[1b0] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 00 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 00 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 01 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 01 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 01 : 53[1b0] -> 57[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 01 : 45[1b0] -> 49[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 01 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 01 : 5[1b0] -> 9[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 01 : 61[1b0] -> 1[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 01 : 53[1b0] -> 57[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 01 : 13[1b0] -> 17[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 01 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 01 : 21[1b0] -> 25[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 01 : 37[1b0] -> 41[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 01 : 45[1b0] -> 49[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 01 : 29[1b0] -> 33[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 01 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 01 : 29[1b0] -> 33[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 01 : 37[1b0] -> 41[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 01 : 13[1b0] -> 17[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 01 : 61[1b0] -> 1[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 00 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 01 : 21[1b0] -> 25[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 01 : 5[1b0] -> 9[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 00 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 00 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 00 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 00 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 00 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 00 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 00 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 00 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 01 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 00 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 01 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 01 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 01 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 01 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 00 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 00 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 01 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 00 : 16[160] -> 35[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 00 : 40[160] -> 51[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 01 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 01 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 00 : 32[160] -> 3[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 00 : 8[160] -> 19[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 01 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 01 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 01 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 01 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 01 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 01 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 01 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 01 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 00 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 00 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 00 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 00 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 01 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 01 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 00 : 48[160] -> 35[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 01 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 01 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 01 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 01 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 00 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 00 : 56[160] -> 51[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 00 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 00 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 01 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 01 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 00 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 00 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 01 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 01 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 01 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 00 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 01 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 01 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 01 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 01 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 01 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 01 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 01 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 00 : 24[160] -> 19[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 00 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 01 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 01 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 01 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 01 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 00 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 01 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 01 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 01 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 01 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 01 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 01 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 01 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 01 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 01 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 01 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 01 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 00 : 24[160] -> 19[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 01 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 00 : 48[160] -> 35[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 00 : 32[160] -> 3[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 01 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 00 : 40[160] -> 51[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 01 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 01 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 01 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 01 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 01 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 01 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 01 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 02 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 01 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 02 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 01 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 02 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 00 : 8[160] -> 19[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 00 : 16[160] -> 35[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 01 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 02 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 00 : 56[160] -> 51[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 02 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 01 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 01 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 00 : 19[190] -> 24[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 02 : 46[1c0] -> 50[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 02 : 22[1c0] -> 26[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 02 : 38[1c0] -> 42[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 00 : 3[190] -> 32[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 02 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 00 : 51[190] -> 40[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 00 : 35[190] -> 48[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 02 : 54[1c0] -> 58[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 02 : 30[1c0] -> 34[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 02 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 01 : 0[160] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 02 : 6[1c0] -> 10[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 01 : 4[1a0] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 02 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 00 : 35[190] -> 16[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 01 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 00 : 19[190] -> 8[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 02 : 14[1c0] -> 18[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 01 : 25[170] -> 18[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 01 : 41[170] -> 50[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 01 : 49[170] -> 34[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 01 : 9[170] -> 18[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 00 : 51[190] -> 56[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 02 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 01 : 57[170] -> 50[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 01 : 33[170] -> 2[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 01 : 17[170] -> 34[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 02 : 62[1c0] -> 2[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 00 : 3[190] -> 32[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 01 : 32[160] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 01 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 01 : 36[1a0] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 01 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 01 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 02 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 01 : 33[170] -> 2[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 02 : 4[1a0] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 02 : 0[160] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 02 : 3[190] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 02 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 02 : 36[1a0] -> 7[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 01 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 00 : 51[190] -> 40[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 02 : 1[170] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 00 : 19[190] -> 8[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 00 : 35[190] -> 16[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 02 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 00 : 51[190] -> 56[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 02 : 5[1b0] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 01 : 40[160] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 00 : 35[190] -> 48[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 01 : 56[160] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 00 : 19[190] -> 24[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 01 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 01 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 01 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 01 : 16[160] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 01 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 01 : 8[160] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 01 : 48[160] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 01 : 24[160] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 01 : 44[1a0] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 01 : 60[1a0] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 01 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 01 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 01 : 41[170] -> 50[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 01 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 01 : 2[180] -> 33[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 01 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 01 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 01 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 01 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 01 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 02 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 01 : 52[1a0] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 01 : 28[1a0] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 01 : 20[1a0] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 02 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 01 : 12[1a0] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 01 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 01 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 01 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 02 : 43[190] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 02 : 38[1c0] -> 42[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 01 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 02 : 59[190] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 01 : 17[170] -> 34[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 02 : 44[1a0] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 02 : 54[1c0] -> 58[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 02 : 60[1a0] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 01 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 02 : 36[1a0] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 02 : 40[160] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 02 : 56[160] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 02 : 32[160] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 02 : 35[190] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 01 : 9[170] -> 18[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 02 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 02 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 02 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 02 : 51[190] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 02 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 02 : 11[190] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 02 : 27[190] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 02 : 6[1c0] -> 10[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 02 : 22[1c0] -> 26[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 02 : 52[1a0] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 02 : 12[1a0] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 02 : 28[1a0] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 02 : 48[160] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 02 : 20[1a0] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 02 : 8[160] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 02 : 24[160] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 02 : 16[160] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 02 : 44[1a0] -> 55[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 02 : 19[190] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 02 : 60[1a0] -> 55[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 02 : 36[1a0] -> 7[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 02 : 20[1a0] -> 39[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 02 : 12[1a0] -> 23[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 02 : 52[1a0] -> 39[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 02 : 44[1a0] -> 55[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 02 : 20[1a0] -> 39[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 02 : 28[1a0] -> 23[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 02 : 12[1a0] -> 23[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 01 : 57[170] -> 50[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 01 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 01 : 50[180] -> 41[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 02 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 02 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 02 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 02 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 01 : 49[170] -> 34[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 02 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 02 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 01 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 01 : 50[180] -> 57[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 01 : 25[170] -> 18[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 02 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 02 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 01 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 01 : 18[180] -> 9[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 01 : 2[180] -> 33[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 01 : 34[180] -> 49[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 02 : 52[1a0] -> 39[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 01 : 18[180] -> 25[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 01 : 34[180] -> 17[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 02 : 33[170] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 02 : 60[1a0] -> 55[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 02 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 02 : 37[1b0] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 02 : 28[1a0] -> 23[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 02 : 62[1c0] -> 2[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 02 : 7[1d0] -> 36[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 02 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 02 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 02 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 02 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 02 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 02 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 02 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 02 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 02 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 02 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 02 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 02 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 02 : 39[1d0] -> 20[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 02 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 02 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 02 : 55[1d0] -> 44[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 02 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 02 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 02 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 02 : 23[1d0] -> 12[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 02 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 02 : 39[1d0] -> 52[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 02 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 02 : 55[1d0] -> 60[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 02 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 02 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 03 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 02 : 23[1d0] -> 28[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 03 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 02 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 01 : 50[180] -> 41[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 02 : 7[1d0] -> 36[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 01 : 50[180] -> 57[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 01 : 34[180] -> 17[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 03 : 7[1d0] -> 11[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 02 : 41[170] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 02 : 57[170] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 01 : 18[180] -> 9[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 02 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 02 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 02 : 46[1c0] -> 50[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 02 : 45[1b0] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 02 : 61[1b0] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 01 : 18[180] -> 25[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 02 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 02 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 02 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 02 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 02 : 9[170] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 02 : 25[170] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 03 : 40[160] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 01 : 34[180] -> 49[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 03 : 42[180] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 02 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 02 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 03 : 56[160] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 03 : 39[1d0] -> 43[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 02 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 03 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 02 : 14[1c0] -> 18[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 03 : 55[1d0] -> 59[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 02 : 29[1b0] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 02 : 13[1b0] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 02 : 17[170] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 03 : 58[180] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 02 : 49[170] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 02 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 02 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 02 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 02 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 02 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 03 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 02 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 02 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 02 : 30[1c0] -> 34[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 02 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 02 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 02 : 21[1b0] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 02 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 02 : 53[1b0] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 03 : 24[160] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 03 : 8[160] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 02 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 03 : 26[180] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 02 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 03 : 10[180] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 03 : 7[1d0] -> 11[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 03 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 03 : 61[1b0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 03 : 23[1d0] -> 27[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 03 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 02 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 03 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 03 : 0[160] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 03 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 03 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 02 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 03 : 63[1d0] -> 3[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 03 : 5[1b0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 03 : 2[180] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 02 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 03 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 03 : 63[1d0] -> 3[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 02 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 02 : 39[1d0] -> 20[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 03 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 03 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 02 : 55[1d0] -> 44[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 03 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 02 : 23[1d0] -> 12[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 02 : 39[1d0] -> 52[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 03 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 02 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 03 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 02 : 55[1d0] -> 60[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 03 : 40[160] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 03 : 56[160] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 03 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 03 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 03 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 02 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 02 : 23[1d0] -> 28[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 03 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 03 : 37[1b0] -> 6[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 02 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 03 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 03 : 39[1d0] -> 43[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 02 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 03 : 55[1d0] -> 59[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 03 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 03 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 03 : 8[160] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 03 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 03 : 60[1a0] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 03 : 24[160] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 03 : 61[1b0] -> 54[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 03 : 23[1d0] -> 27[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 03 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 03 : 0[160] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 03 : 4[1a0] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 03 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 02 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 02 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 02 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 03 : 45[1b0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 03 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 02 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 03 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 02 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 02 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 03 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 03 : 47[1d0] -> 51[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 02 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 02 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 02 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 03 : 44[1a0] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 03 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 03 : 48[160] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 02 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 03 : 45[1b0] -> 54[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 03 : 50[180] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 03 : 53[1b0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 03 : 47[1d0] -> 51[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 03 : 13[1b0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 03 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 02 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 02 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 02 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 03 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 03 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 03 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 03 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 02 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 03 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 03 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 03 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 03 : 29[1b0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 03 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 03 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 03 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 03 : 15[1d0] -> 19[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 02 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 02 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 03 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 03 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 03 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 03 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 02 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 03 : 12[1a0] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 03 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 04 : 4[1a0] -> 8[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 03 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 03 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 03 : 53[1b0] -> 38[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 03 : 13[1b0] -> 22[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 03 : 45[1b0] -> 54[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 03 : 16[160] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 04 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 03 : 15[1d0] -> 19[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 02 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 03 : 31[1d0] -> 35[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 03 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 03 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 03 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 03 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 03 : 21[1b0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 03 : 18[180] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 03 : 32[160] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 03 : 28[1a0] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 03 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 03 : 29[1b0] -> 22[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 03 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 04 : 60[1a0] -> 0[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 03 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 03 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 03 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 03 : 31[1d0] -> 35[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 03 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 03 : 37[1b0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 03 : 34[180] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 04 : 41[170] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 04 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 04 : 9[170] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 04 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 04 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 04 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 03 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 03 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 04 : 36[1a0] -> 40[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 04 : 4[1a0] -> 8[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 03 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 03 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 04 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 04 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 04 : 1[170] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 04 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 03 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 04 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 04 : 60[1a0] -> 0[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 04 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 04 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 03 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 03 : 21[1b0] -> 38[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 04 : 57[170] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 03 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 04 : 25[170] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 04 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 03 : 13[1b0] -> 22[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 04 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 04 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 04 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 04 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 04 : 52[1a0] -> 56[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 03 : 48[160] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 04 : 20[1a0] -> 24[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 03 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 04 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 04 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 04 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 04 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 03 : 37[1b0] -> 6[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 03 : 21[1b0] -> 38[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 04 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 04 : 40[160] -> 11[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 04 : 24[160] -> 43[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 03 : 52[1a0] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 03 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 04 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 04 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 05 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 04 : 0[160] -> 59[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 04 : 16[160] -> 27[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 04 : 48[160] -> 59[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 03 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 04 : 52[1a0] -> 56[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 03 : 16[160] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 03 : 61[1b0] -> 54[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 03 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 03 : 20[1a0] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 03 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 04 : 56[160] -> 43[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 03 : 32[160] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 03 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 03 : 36[1a0] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 04 : 20[1a0] -> 24[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 03 : 29[1b0] -> 22[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 04 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 04 : 36[1a0] -> 40[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 04 : 24[160] -> 43[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 05 : 8[160] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 03 : 53[1b0] -> 38[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 04 : 56[160] -> 43[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 04 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 03 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 04 : 40[160] -> 11[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 04 : 32[160] -> 27[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 04 : 0[160] -> 59[190] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 04 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 03 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 03 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 04 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 03 : 6[1c0] -> 37[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 04 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 03 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 04 : 11[190] -> 40[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 04 : 43[190] -> 24[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 03 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 03 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 03 : 54[1c0] -> 45[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 04 : 59[190] -> 0[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 03 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 04 : 43[190] -> 56[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 04 : 44[1a0] -> 48[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 03 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 03 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 03 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 03 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 04 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 04 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 04 : 49[170] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 04 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 03 : 54[1c0] -> 61[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 04 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 03 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 04 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 04 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 04 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 03 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 04 : 44[1a0] -> 48[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 04 : 11[190] -> 40[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 03 : 6[1c0] -> 37[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 03 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 03 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 04 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 04 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 04 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 04 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 04 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 04 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 04 : 12[1a0] -> 16[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 04 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 04 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 05 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 03 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 03 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 04 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 03 : 22[1c0] -> 13[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 05 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 05 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 05 : 40[160] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 05 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 05 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 04 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 03 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 04 : 28[1a0] -> 32[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 04 : 48[160] -> 59[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 04 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 03 : 22[1c0] -> 29[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 03 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 04 : 17[170] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 03 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 04 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 04 : 5[1b0] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 04 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 04 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 03 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 04 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 03 : 38[1c0] -> 21[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 04 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 04 : 12[1a0] -> 16[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 03 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 04 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 04 : 59[190] -> 48[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 04 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 04 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 04 : 33[170] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 04 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 03 : 38[1c0] -> 53[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 05 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 04 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 04 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 04 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 03 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 04 : 28[1a0] -> 32[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 05 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 04 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 05 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 05 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 05 : 61[1b0] -> 1[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 04 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 04 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 05 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 04 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 05 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 05 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 04 : 16[160] -> 27[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 05 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 04 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 05 : 5[1b0] -> 9[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 05 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 05 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 05 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 05 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 04 : 32[160] -> 27[190] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 04 : 27[190] -> 16[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 04 : 27[190] -> 32[160] [receive] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 03 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 04 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 04 : 43[190] -> 24[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 04 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 03 : 54[1c0] -> 45[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 04 : 43[190] -> 56[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 03 : 54[1c0] -> 61[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 05 : 24[160] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 05 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 05 : 56[160] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 03 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 05 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 04 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 04 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 04 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 04 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 04 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 04 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 03 : 22[1c0] -> 13[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 03 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 05 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 04 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 04 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 04 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 04 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 04 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 04 : 61[1b0] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 04 : 45[1b0] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 04 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 03 : 22[1c0] -> 29[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 04 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 03 : 38[1c0] -> 21[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 04 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 05 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 05 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 05 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 05 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 05 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 05 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 04 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 04 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 04 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 05 : 53[1b0] -> 57[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 05 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 03 : 38[1c0] -> 53[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 04 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 04 : 59[190] -> 48[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 05 : 37[1b0] -> 41[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 05 : 60[1a0] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 05 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 05 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 04 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 04 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 04 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 05 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 04 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 05 : 61[1b0] -> 1[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 05 : 44[1a0] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 04 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 05 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 04 : 13[1b0] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 04 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 04 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 04 : 29[1b0] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 05 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 05 : 45[1b0] -> 49[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 05 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 05 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 05 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 04 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 04 : 37[1b0] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 06 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 04 : 59[190] -> 0[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 05 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 04 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 04 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 04 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 05 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 05 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 04 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 05 : 1[170] -> 58[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 05 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 06 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 04 : 21[1b0] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 04 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 05 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 05 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 04 : 53[1b0] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 05 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 05 : 5[1b0] -> 9[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 04 : 27[190] -> 16[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 05 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 05 : 0[160] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 05 : 21[1b0] -> 25[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 05 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 06 : 62[1c0] -> 2[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 05 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 05 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 05 : 28[1a0] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 05 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 05 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 06 : 40[160] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 05 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 05 : 29[1b0] -> 33[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 05 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 05 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 06 : 44[1a0] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 05 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 05 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 05 : 48[160] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 05 : 12[1a0] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 05 : 29[1b0] -> 33[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 05 : 13[1b0] -> 17[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 05 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 05 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 05 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 05 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 05 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 06 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 05 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 04 : 27[190] -> 32[160] [send] via NET/AWS Libfabric/0 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 05 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 05 : 37[1b0] -> 41[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 05 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 05 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 05 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 05 : 25[170] -> 42[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 06 : 43[190] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 05 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 05 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 05 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 05 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 05 : 13[1b0] -> 17[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 05 : 45[1b0] -> 49[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 05 : 49[170] -> 58[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 06 : 60[1a0] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 05 : 4[1a0] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 06 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 05 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 05 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 05 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 05 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 05 : 21[1b0] -> 25[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 06 : 56[160] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 05 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 06 : 59[190] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 05 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 05 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 05 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 06 : 44[1a0] -> 15[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 05 : 16[160] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 05 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 05 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 06 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 06 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 06 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 05 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 05 : 52[1a0] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 05 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 06 : 8[160] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 05 : 41[170] -> 10[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 05 : 53[1b0] -> 57[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 06 : 28[1a0] -> 47[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 05 : 32[160] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 05 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 05 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 06 : 12[1a0] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 05 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 06 : 60[1a0] -> 47[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 06 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 06 : 46[1c0] -> 50[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 05 : 33[170] -> 26[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 06 : 62[1c0] -> 2[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 06 : 30[1c0] -> 34[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 06 : 52[1a0] -> 63[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 06 : 6[1c0] -> 10[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 06 : 3[190] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 05 : 41[170] -> 10[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 06 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 05 : 17[170] -> 26[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 06 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 06 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 05 : 25[170] -> 42[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 06 : 4[1a0] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 05 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 05 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 06 : 11[190] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 06 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 06 : 0[160] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 05 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 05 : 20[1a0] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 05 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 06 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 06 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 05 : 57[170] -> 42[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 05 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 06 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 05 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 05 : 36[1a0] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 05 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 05 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 06 : 14[1c0] -> 18[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 06 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 05 : 17[170] -> 26[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 05 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 06 : 38[1c0] -> 42[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 06 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 06 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 06 : 28[1a0] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 05 : 49[170] -> 58[180] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 06 : 52[1a0] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 06 : 44[1a0] -> 15[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 06 : 24[160] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 06 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 06 : 4[1a0] -> 63[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 06 : 27[190] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 06 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 06 : 22[1c0] -> 26[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 06 : 19[190] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 06 : 20[1a0] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 06 : 36[1a0] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 06 : 35[190] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 06 : 14[1c0] -> 18[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 06 : 30[1c0] -> 34[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 06 : 16[160] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 06 : 32[160] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 06 : 48[160] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 06 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 06 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 06 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 06 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 06 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 06 : 28[1a0] -> 47[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 06 : 51[190] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 05 : 57[170] -> 42[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 05 : 1[170] -> 58[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 06 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 06 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 06 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 05 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 06 : 20[1a0] -> 31[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 05 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 06 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 06 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 06 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 06 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 06 : 46[1c0] -> 50[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 06 : 36[1a0] -> 31[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 06 : 20[1a0] -> 31[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 06 : 54[1c0] -> 58[180] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 05 : 58[180] -> 49[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 06 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 06 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 06 : 52[1a0] -> 63[1d0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 06 : 60[1a0] -> 47[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 06 : 47[1d0] -> 28[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 05 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 06 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 05 : 42[180] -> 57[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 06 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 06 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 06 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 05 : 58[180] -> 1[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 06 : 4[1a0] -> 63[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 06 : 9[170] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 06 : 63[1d0] -> 52[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 06 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 05 : 10[180] -> 41[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 06 : 47[1d0] -> 60[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 06 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 06 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 06 : 13[1b0] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 05 : 33[170] -> 26[180] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 05 : 26[180] -> 17[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 06 : 15[1d0] -> 44[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 05 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 06 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 05 : 26[180] -> 33[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 05 : 42[180] -> 25[170] [receive] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 05 : 10[180] -> 41[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 06 : 36[1a0] -> 31[1d0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 06 : 31[1d0] -> 20[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 06 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 06 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 06 : 41[170] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 07 : 12[1a0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 07 : 14[1c0] -> 13[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 06 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 06 : 6[1c0] -> 10[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 06 : 15[1d0] -> 44[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 06 : 31[1d0] -> 36[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 06 : 45[1b0] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 06 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 06 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 06 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 06 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 06 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 06 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 06 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 06 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 06 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 06 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 07 : 15[1d0] -> 19[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 06 : 63[1d0] -> 4[1a0] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 06 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 06 : 63[1d0] -> 52[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 07 : 13[1b0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 07 : 8[160] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 07 : 10[180] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 07 : 7[1d0] -> 11[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO Ring 07 : 9[170] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 07 : 7[1d0] -> 11[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 06 : 63[1d0] -> 4[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 07 : 11[190] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO Ring 07 : 13[1b0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 07 : 10[180] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 07 : 4[1a0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 07 : 8[160] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 07 : 7[1d0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 05 : 42[180] -> 25[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 05 : 58[180] -> 49[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 07 : 45[1b0] -> 14[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 07 : 63[1d0] -> 3[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 06 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO Ring 07 : 11[190] -> 10[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO Ring 07 : 10[180] -> 9[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO Ring 07 : 8[160] -> 11[190] via P2P/IPC tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO Ring 07 : 12[1a0] -> 8[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 07 : 44[1a0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 05 : 58[180] -> 1[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 07 : 46[1c0] -> 45[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 05 : 42[180] -> 57[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-1:15:577 [1] NCCL INFO comm 0x7f27103b5d90 rank 9 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-1:17:595 [3] NCCL INFO comm 0x7efe103ad830 rank 11 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-1:16:589 [2] NCCL INFO comm 0x7fda4c3b4ba0 rank 10 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-1:14:574 [0] NCCL INFO comm 0x7f241c5a46d0 rank 8 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 06 : 47[1d0] -> 28[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 06 : 49[170] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 06 : 1[170] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 06 : 25[170] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 06 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 06 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 06 : 57[170] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 06 : 47[1d0] -> 60[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 06 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 06 : 54[1c0] -> 58[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 06 : 38[1c0] -> 42[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 06 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 06 : 53[1b0] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 06 : 5[1b0] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 06 : 29[1b0] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 06 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 06 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 06 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 06 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 06 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 06 : 61[1b0] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 05 : 26[180] -> 17[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 06 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 06 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 06 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 06 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 07 : 60[1a0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 06 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 07 : 6[1c0] -> 5[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 06 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 07 : 0[160] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 07 : 47[1d0] -> 51[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 06 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 06 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 07 : 63[1d0] -> 3[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 06 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 06 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 07 : 62[1c0] -> 61[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 06 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 06 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 05 : 26[180] -> 33[170] [send] via NET/AWS Libfabric/1 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 07 : 39[1d0] -> 43[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 07 : 28[1a0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 07 : 30[1c0] -> 29[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 07 : 48[160] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 07 : 2[180] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 07 : 5[1b0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 07 : 3[190] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 06 : 31[1d0] -> 20[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 06 : 33[170] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 06 : 17[170] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 07 : 47[1d0] -> 51[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 07 : 63[1d0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 06 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 06 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 07 : 45[1b0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 07 : 40[160] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 06 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 07 : 42[180] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 06 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 06 : 22[1c0] -> 26[180] [receive] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO Ring 07 : 1[170] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 06 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 07 : 39[1d0] -> 43[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 06 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 06 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 06 : 31[1d0] -> 36[1a0] [send] via NET/AWS Libfabric/2 tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 07 : 51[190] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 06 : 37[1b0] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 06 : 21[1b0] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 06 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO Ring 07 : 41[170] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 07 : 56[160] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 07 : 61[1b0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 07 : 58[180] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 06 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 07 : 50[180] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 06 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 06 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 07 : 55[1d0] -> 59[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 06 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 07 : 0[160] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 07 : 47[1d0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 07 : 2[180] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO Ring 07 : 57[170] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO Ring 07 : 49[170] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 07 : 43[190] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 06 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 07 : 38[1c0] -> 37[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 07 : 59[190] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 07 : 52[1a0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 07 : 42[180] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 06 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 07 : 5[1b0] -> 62[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 07 : 32[160] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 06 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 07 : 36[1a0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 06 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 07 : 16[160] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 07 : 58[180] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 07 : 31[1d0] -> 35[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO Ring 07 : 6[1c0] -> 7[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO Ring 07 : 7[1d0] -> 4[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 07 : 53[1b0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 07 : 48[160] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 07 : 40[160] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 06 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 07 : 15[1d0] -> 19[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 06 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 07 : 54[1c0] -> 53[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 07 : 56[160] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 07 : 31[1d0] -> 35[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO Ring 07 : 4[1a0] -> 0[160] via P2P/IPC tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Ring 07 : 0[160] -> 3[190] via P2P/IPC tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 07 : 61[1b0] -> 46[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO Ring 07 : 3[190] -> 2[180] via P2P/IPC tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO Ring 07 : 2[180] -> 1[170] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 07 : 15[1d0] -> 14[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 07 : 45[1b0] -> 14[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 07 : 29[1b0] -> 46[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO Ring 07 : 47[1d0] -> 44[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 07 : 37[1b0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO Ring 07 : 63[1d0] -> 60[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 07 : 35[190] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 07 : 39[1d0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 07 : 34[180] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 07 : 29[1b0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:21:574 [7] NCCL INFO comm 0x7f3a583999b0 rank 7 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 07 : 24[160] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 07 : 53[1b0] -> 62[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 07 : 26[180] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO Ring 07 : 15[1d0] -> 12[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-0:15:595 [1] NCCL INFO comm 0x7f1da8389ae0 rank 1 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 07 : 19[190] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 07 : 18[180] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 07 : 50[180] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-0:18:578 [4] NCCL INFO comm 0x7f4984393090 rank 4 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 07 : 55[1d0] -> 59[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO comm 0x7fb8bc5de7f0 rank 0 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-1:18:583 [4] NCCL INFO comm 0x7fcb543ae3c0 rank 12 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-0:17:579 [3] NCCL INFO comm 0x7f501c3cddd0 rank 3 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-0:16:577 [2] NCCL INFO comm 0x7f024c393c80 rank 2 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO Ring 07 : 60[1a0] -> 56[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO Ring 07 : 56[160] -> 59[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 07 : 23[1d0] -> 27[190] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO Ring 07 : 33[170] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO Ring 07 : 43[190] -> 42[180] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO Ring 07 : 42[180] -> 41[170] via P2P/IPC tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO Ring 07 : 40[160] -> 43[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO Ring 07 : 44[1a0] -> 40[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:18:592 [4] NCCL INFO comm 0x7f1dc43c5610 rank 60 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 07 : 20[1a0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO Ring 07 : 17[170] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO Ring 07 : 25[170] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 07 : 31[1d0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO Ring 07 : 59[190] -> 58[180] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO Ring 07 : 58[180] -> 57[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 07 : 21[1b0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:15:579 [1] NCCL INFO comm 0x7fdf703c7b80 rank 41 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 07 : 14[1c0] -> 45[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 07 : 53[1b0] -> 62[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:14:576 [0] NCCL INFO comm 0x7fc7d4ef3860 rank 56 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:17:593 [3] NCCL INFO comm 0x7fa5303ccff0 rank 43 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 07 : 32[160] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 07 : 55[1d0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:16:576 [2] NCCL INFO comm 0x7fe10c3c7990 rank 42 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:14:577 [0] NCCL INFO comm 0x7fcb5c9bdc10 rank 40 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO Ring 07 : 50[180] -> 49[170] via P2P/IPC tensorflow-benchmarks-efa-worker-7:15:578 [1] NCCL INFO comm 0x7f05443c1920 rank 57 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 07 : 34[180] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-5:18:592 [4] NCCL INFO comm 0x7f12f03c7c20 rank 44 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO Ring 07 : 45[1b0] -> 46[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 07 : 22[1c0] -> 21[1b0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:17:577 [3] NCCL INFO comm 0x7fbca83c1760 rank 59 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO Ring 07 : 51[190] -> 50[180] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO Ring 07 : 52[1a0] -> 48[160] via P2P/IPC tensorflow-benchmarks-efa-worker-7:16:574 [2] NCCL INFO comm 0x7fa2483c15d0 rank 58 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 07 : 16[160] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO Ring 07 : 48[160] -> 51[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 07 : 27[190] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-6:15:576 [1] NCCL INFO comm 0x7f6e0c3c0470 rank 49 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 07 : 26[180] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 07 : 18[180] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 07 : 37[1b0] -> 30[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 07 : 14[1c0] -> 15[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:16:579 [2] NCCL INFO comm 0x7f4a6c3bfd00 rank 50 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-6:17:574 [3] NCCL INFO comm 0x7fdeb83c0040 rank 51 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-1:19:591 [5] NCCL INFO comm 0x7f24b03ae100 rank 13 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 07 : 23[1d0] -> 27[190] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:14:578 [0] NCCL INFO comm 0x7fba79685ca0 rank 48 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-1:21:582 [7] NCCL INFO comm 0x7f28183a5180 rank 15 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 07 : 24[160] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO Ring 07 : 55[1d0] -> 52[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO Ring 07 : 54[1c0] -> 55[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 07 : 29[1b0] -> 46[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO Ring 07 : 38[1c0] -> 39[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO Ring 07 : 39[1d0] -> 36[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:18:585 [4] NCCL INFO comm 0x7f215c3c0450 rank 52 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-6:21:577 [7] NCCL INFO comm 0x7f66a03c7da0 rank 55 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO Ring 07 : 31[1d0] -> 28[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 07 : 21[1b0] -> 30[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO Ring 07 : 32[160] -> 35[190] via P2P/IPC tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO Ring 07 : 36[1a0] -> 32[160] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO Ring 07 : 14[1c0] -> 45[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO Ring 07 : 34[180] -> 33[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO Ring 07 : 35[190] -> 34[180] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 07 : 21[1b0] -> 30[1c0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:21:575 [7] NCCL INFO comm 0x7f150c3d6fd0 rank 39 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO Ring 07 : 24[160] -> 27[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO Ring 07 : 28[1a0] -> 24[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO Ring 07 : 18[180] -> 17[170] via P2P/IPC tensorflow-benchmarks-efa-worker-4:15:574 [1] NCCL INFO comm 0x7fa4883c72b0 rank 33 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 07 : 23[1d0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO Ring 07 : 19[190] -> 18[180] via P2P/IPC tensorflow-benchmarks-efa-worker-1:20:580 [6] NCCL INFO comm 0x7ff5083ae0b0 rank 14 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:14:586 [0] NCCL INFO comm 0x7fedb4878780 rank 32 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:18:580 [4] NCCL INFO comm 0x7fd2b83c82f0 rank 36 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:16:595 [2] NCCL INFO comm 0x7f45303d1210 rank 34 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:17:592 [3] NCCL INFO comm 0x7f32043c7b50 rank 35 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO Ring 07 : 20[1a0] -> 16[160] via P2P/IPC tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO Ring 07 : 16[160] -> 19[190] via P2P/IPC tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO Ring 07 : 27[190] -> 26[180] via P2P/IPC tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO Ring 07 : 26[180] -> 25[170] via P2P/IPC tensorflow-benchmarks-efa-worker-2:15:595 [1] NCCL INFO comm 0x7fd7183c4a00 rank 17 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:19:596 [4] NCCL INFO comm 0x7fc72c3c7a40 rank 28 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:15:575 [0] NCCL INFO comm 0x7f4f7c940b50 rank 24 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:16:576 [1] NCCL INFO comm 0x7f0a943c2b70 rank 25 nranks 64 cudaDev 1 busId 170 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 07 : 61[1b0] -> 46[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:16:589 [2] NCCL INFO comm 0x7fb6a03c7da0 rank 18 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:17:580 [2] NCCL INFO comm 0x7fd2243c3900 rank 26 nranks 64 cudaDev 2 busId 180 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:17:574 [3] NCCL INFO comm 0x7f59483b6aa0 rank 19 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:18:593 [3] NCCL INFO comm 0x7f03f43c7e50 rank 27 nranks 64 cudaDev 3 busId 190 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:14:576 [0] NCCL INFO comm 0x7fe131679ee0 rank 16 nranks 64 cudaDev 0 busId 160 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 07 : 46[1c0] -> 29[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 07 : 62[1c0] -> 53[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 07 : 5[1b0] -> 62[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO Ring 07 : 23[1d0] -> 20[1a0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO Ring 07 : 22[1c0] -> 23[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO Ring 07 : 29[1b0] -> 30[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO Ring 07 : 53[1b0] -> 54[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:18:575 [4] NCCL INFO comm 0x7f5b283bb550 rank 20 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE tensorflow-benchmarks-efa-worker-2:21:580 [7] NCCL INFO comm 0x7fa3303ce340 rank 23 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-6:20:595 [6] NCCL INFO comm 0x7f17603c0090 rank 54 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 07 : 46[1c0] -> 61[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO Ring 07 : 61[1b0] -> 62[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 07 : 30[1c0] -> 21[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 07 : 37[1b0] -> 30[1c0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO Ring 07 : 21[1b0] -> 22[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-2:20:584 [6] NCCL INFO comm 0x7f79503ba180 rank 22 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 07 : 30[1c0] -> 37[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO Ring 07 : 37[1b0] -> 38[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-4:20:581 [6] NCCL INFO comm 0x7f1df43c7df0 rank 38 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 07 : 62[1c0] -> 63[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:21:579 [7] NCCL INFO comm 0x7f61b83d11a0 rank 63 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 07 : 62[1c0] -> 5[1b0] [receive] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO Ring 07 : 5[1b0] -> 6[1c0] via P2P/IPC tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 07 : 62[1c0] -> 53[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-0:20:576 [6] NCCL INFO comm 0x7f73fc3936d0 rank 6 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO Ring 07 : 62[1c0] -> 5[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-6:19:575 [5] NCCL INFO comm 0x7f6bb83c6130 rank 53 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-7:20:575 [6] NCCL INFO comm 0x7fbbfc3cfcb0 rank 62 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-0:19:575 [5] NCCL INFO comm 0x7ffa14392bc0 rank 5 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 07 : 46[1c0] -> 47[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-5:19:589 [5] NCCL INFO comm 0x7ff5403ce080 rank 45 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:21:574 [7] NCCL INFO comm 0x7f3bb03c88f0 rank 47 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 07 : 46[1c0] -> 29[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO Ring 07 : 46[1c0] -> 61[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-7:19:595 [5] NCCL INFO comm 0x7f1cc03c4530 rank 61 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-5:20:575 [6] NCCL INFO comm 0x7f50043c7600 rank 46 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 07 : 30[1c0] -> 31[1d0] via P2P/IPC tensorflow-benchmarks-efa-worker-3:20:584 [5] NCCL INFO comm 0x7f78443c78c0 rank 29 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:22:581 [7] NCCL INFO comm 0x7fbee83ca670 rank 31 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 07 : 30[1c0] -> 21[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO Ring 07 : 30[1c0] -> 37[1b0] [send] via NET/AWS Libfabric/3 tensorflow-benchmarks-efa-worker-2:19:591 [5] NCCL INFO comm 0x7f2c683b9600 rank 21 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-4:19:589 [5] NCCL INFO comm 0x7f2d9c3c7a40 rank 37 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE tensorflow-benchmarks-efa-worker-3:21:577 [6] NCCL INFO comm 0x7fd9343c43d0 rank 30 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE tensorflow-benchmarks-efa-worker-0:14:584 [0] NCCL INFO Launch mode Parallel Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Done warm up Done warm up Step Img/sec total_loss Step Img/sec total_loss Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Step Img/sec total_loss Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Done warm up Done warm up Step Img/sec total_loss Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss Step Img/sec total_loss Done warm up Step Img/sec total_loss Done warm up Step Img/sec total_loss 1 images/sec: 187.5 +/- 0.0 (jitter = 0.0) 8.785 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.558 1 images/sec: 187.3 +/- 0.0 (jitter = 0.0) 8.335 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.387 1 images/sec: 187.3 +/- 0.0 (jitter = 0.0) 8.388 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.282 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.445 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.445 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.402 1 images/sec: 187.3 +/- 0.0 (jitter = 0.0) 8.286 1 images/sec: 186.9 +/- 0.0 (jitter = 0.0) 8.383 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.270 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.538 1 images/sec: 186.9 +/- 0.0 (jitter = 0.0) 8.334 1 images/sec: 186.8 +/- 0.0 (jitter = 0.0) 8.507 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.282 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.496 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.249 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.377 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.294 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.548 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.325 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.615 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.486 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.499 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.426 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.352 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.526 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.277 1 images/sec: 186.7 +/- 0.0 (jitter = 0.0) 8.507 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.236 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.431 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.472 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.267 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.382 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.319 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.189 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.550 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.335 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.298 1 images/sec: 186.9 +/- 0.0 (jitter = 0.0) 8.388 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.401 1 images/sec: 187.3 +/- 0.0 (jitter = 0.0) 8.503 1 images/sec: 186.9 +/- 0.0 (jitter = 0.0) 8.202 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.405 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.409 1 images/sec: 187.4 +/- 0.0 (jitter = 0.0) 8.288 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.673 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.427 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.684 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.772 1 images/sec: 187.0 +/- 0.0 (jitter = 0.0) 8.402 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.252 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.472 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.133 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.453 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.371 1 images/sec: 187.2 +/- 0.0 (jitter = 0.0) 8.373 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.281 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.298 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.447 1 images/sec: 187.1 +/- 0.0 (jitter = 0.0) 8.323 1 images/sec: 186.8 +/- 0.0 (jitter = 0.0) 8.477 1 images/sec: 186.6 +/- 0.0 (jitter = 0.0) 8.263 10 images/sec: 187.4 +/- 0.3 (jitter = 1.0) 8.867 10 images/sec: 187.4 +/- 0.2 (jitter = 0.8) 8.163 10 images/sec: 187.4 +/- 0.3 (jitter = 1.0) 8.489 10 images/sec: 187.4 +/- 0.2 (jitter = 0.8) 8.259 10 images/sec: 187.4 +/- 0.3 (jitter = 0.8) 8.980 10 images/sec: 187.4 +/- 0.2 (jitter = 0.9) 10.794 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 9.937 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 8.307 10 images/sec: 187.4 +/- 0.3 (jitter = 0.8) 8.221 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 8.038 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 8.167 10 images/sec: 187.4 +/- 0.3 (jitter = 1.1) 11.249 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 8.290 10 images/sec: 187.4 +/- 0.3 (jitter = 0.9) 8.308 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 8.856 10 images/sec: 187.3 +/- 0.2 (jitter = 0.7) 8.205 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 8.208 10 images/sec: 187.4 +/- 0.3 (jitter = 0.8) 10.533 10 images/sec: 187.4 +/- 0.2 (jitter = 0.7) 8.189 10 images/sec: 187.4 +/- 0.3 (jitter = 0.8) 8.163 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 8.457 10 images/sec: 187.4 +/- 0.2 (jitter = 0.7) 8.384 10 images/sec: 187.4 +/- 0.3 (jitter = 0.6) 8.284 10 images/sec: 187.4 +/- 0.2 (jitter = 0.8) 8.061 10 images/sec: 187.3 +/- 0.2 (jitter = 0.6) 8.150 10 images/sec: 187.3 +/- 0.2 (jitter = 0.4) 8.255 10 images/sec: 187.3 +/- 0.2 (jitter = 0.5) 8.267 10 images/sec: 187.3 +/- 0.2 (jitter = 0.6) 8.269 10 images/sec: 187.3 +/- 0.2 (jitter = 0.4) 8.269 10 images/sec: 187.3 +/- 0.2 (jitter = 0.4) 9.191 10 images/sec: 187.2 +/- 0.2 (jitter = 0.5) 8.175 10 images/sec: 187.4 +/- 0.3 (jitter = 0.5) 8.492 10 images/sec: 187.4 +/- 0.2 (jitter = 0.6) 8.688 10 images/sec: 187.4 +/- 0.2 (jitter = 0.7) 8.164 10 images/sec: 187.3 +/- 0.3 (jitter = 0.9) 10.706 10 images/sec: 187.3 +/- 0.3 (jitter = 0.8) 8.795 10 images/sec: 187.3 +/- 0.3 (jitter = 0.8) 8.324 10 images/sec: 187.3 +/- 0.3 (jitter = 0.7) 8.529 10 images/sec: 187.4 +/- 0.3 (jitter = 0.7) 10.688 10 images/sec: 187.2 +/- 0.2 (jitter = 0.5) 8.004 10 images/sec: 187.3 +/- 0.3 (jitter = 1.0) 8.689 10 images/sec: 187.4 +/- 0.3 (jitter = 1.1) 8.078 10 images/sec: 187.3 +/- 0.2 (jitter = 0.8) 8.218 10 images/sec: 187.4 +/- 0.3 (jitter = 1.1) 8.164 10 images/sec: 187.3 +/- 0.2 (jitter = 0.7) 8.155 10 images/sec: 187.3 +/- 0.2 (jitter = 0.7) 8.353 10 images/sec: 187.3 +/- 0.3 (jitter = 0.9) 8.216 10 images/sec: 187.3 +/- 0.3 (jitter = 0.5) 8.306 10 images/sec: 187.3 +/- 0.2 (jitter = 0.7) 8.097 10 images/sec: 187.3 +/- 0.2 (jitter = 0.7) 8.362 10 images/sec: 187.3 +/- 0.2 (jitter = 0.8) 8.230 10 images/sec: 187.3 +/- 0.2 (jitter = 0.6) 8.259 10 images/sec: 187.3 +/- 0.3 (jitter = 0.9) 8.602 10 images/sec: 187.3 +/- 0.2 (jitter = 0.8) 8.197 10 images/sec: 187.3 +/- 0.2 (jitter = 0.9) 8.383 10 images/sec: 187.3 +/- 0.2 (jitter = 0.6) 8.445 10 images/sec: 187.3 +/- 0.3 (jitter = 0.8) 8.276 10 images/sec: 187.3 +/- 0.2 (jitter = 0.7) 8.378 10 images/sec: 187.3 +/- 0.2 (jitter = 0.7) 8.095 10 images/sec: 187.3 +/- 0.2 (jitter = 0.8) 9.506 10 images/sec: 187.3 +/- 0.3 (jitter = 1.0) 9.318 10 images/sec: 187.3 +/- 0.2 (jitter = 1.0) 10.401 10 images/sec: 187.3 +/- 0.2 (jitter = 0.6) 8.784 10 images/sec: 187.3 +/- 0.3 (jitter = 0.6) 8.648 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.240 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 9.883 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.170 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.207 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.310 20 images/sec: 187.5 +/- 0.1 (jitter = 0.6) 8.176 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.272 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.344 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.329 20 images/sec: 187.5 +/- 0.1 (jitter = 0.4) 8.213 20 images/sec: 187.5 +/- 0.2 (jitter = 0.7) 8.304 20 images/sec: 187.5 +/- 0.1 (jitter = 0.6) 8.287 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.356 20 images/sec: 187.5 +/- 0.2 (jitter = 0.6) 8.404 2020 images/sec: 187.5 +/- 0.1 (jitter = 0.6) 8.319 20 images/sec: 187.5 +/- 0.2 (jitter = 0.7) 8.209 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.366 20 images/sec: 187.5 +/- 0.1 (jitter = 0.5) 8.287 20 images/sec: 187.5 +/- 0.1 (jitter = 0.5) 8.125 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 9.966 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.030 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 8.454 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.033 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.124 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.473 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.266 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.139 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 12.605 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.471 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.113 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.742 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 8.222 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.157 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.145 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.081 20 images/sec: 187.5 +/- 0.2 (jitter = 0.7) 8.121 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 8.317 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 8.333 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 8.576 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 11.535 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.721 20 images/sec: 187.5 +/- 0.2 (jitter = 1.2) 8.247 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.454 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.300 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.182 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 12.797 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.155 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 8.133 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.335 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.391 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.150 20 images/sec: 187.5 +/- 0.2 (jitter = 1.1) 8.202 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.360 20 images/sec: 187.4 +/- 0.2 (jitter = 0.8) 8.466 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.184 20 images/sec: 187.4 +/- 0.2 (jitter = 0.9) 8.325 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.287 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.572 20 images/sec: 187.5 +/- 0.2 (jitter = 0.7) 8.683 20 images/sec: 187.5 +/- 0.2 (jitter = 1.0) 8.241 20 images/sec: 187.5 +/- 0.2 (jitter = 0.9) 8.177 20 images/sec: 187.4 +/- 0.2 (jitter = 0.9) 8.205 20 images/sec: 187.4 +/- 0.2 (jitter = 1.0) 9.693 20 images/sec: 187.5 +/- 0.2 (jitter = 0.8) 8.430 30 images/sec: 187.8 +/- 0.1 (jitter = 0.9) 8.015 30 images/sec: 187.8 +/- 0.1 (jitter = 0.9) 8.564 30 images/sec: 187.8 +/- 0.1 (jitter = 1.0) 8.000 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 7.997 30 images/sec: 187.8 +/- 0.1 (jitter = 0.9) 8.040 30 images/sec: 187.8 +/- 0.1 (jitter = 1.0) 8.031 30 images/sec: 187.8 +/- 0.1 (jitter = 1.0) 8.035 30 images/sec: 187.8 +/- 0.1 (jitter = 0.9) 8.051 30 images/sec: 187.8 +/- 0.1 (jitter = 0.8) 8.014 30 images/sec: 187.8 +/- 0.1 (jitter = 0.6) 7.967 30 images/sec: 187.8 +/- 0.1 (jitter = 0.6) 8.011 30 images/sec: 187.8 +/- 0.1 (jitter = 1.0) 8.059 30 images/sec: 187.8 +/- 0.1 (jitter = 0.6) 8.020 30 images/sec: 187.8 +/- 0.1 (jitter = 0.9) 8.037 30 images/sec: 187.8 +/- 0.1 (jitter = 0.6) 7.929 30 images/sec: 187.8 +/- 0.1 (jitter = 0.7) 8.035 30 images/sec: 187.8 +/- 0.1 (jitter = 0.8) 8.036 30 images/sec: 187.8 +/- 0.1 (jitter = 0.7) 7.963 30 images/sec: 187.8 +/- 0.1 (jitter = 0.7) 7.990 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.066 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.076 30 images/sec: 187.8 +/- 0.1 (jitter = 0.7) 7.985 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.023 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.049 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.041 30 images/sec: 187.8 +/- 0.1 (jitter = 0.8) 7.978 30 images/sec: 187.8 +/- 0.1 (jitter = 0.7) 8.056 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.098 30 images/sec: 187.8 +/- 0.1 (jitter = 0.8) 8.004 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.029 30 images/sec: 187.8 +/- 0.2 (jitter = 1.1) 7.996 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.454 30 images/sec: 187.8 +/- 0.2 (jitter = 1.1) 7.976 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.085 30 images/sec: 187.8 +/- 0.2 (jitter = 1.1) 8.040 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 7.999 30 images/sec: 187.8 +/- 0.2 (jitter = 1.1) 8.237 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 7.989 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.099 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.011 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 7.957 30 images/sec: 187.8 +/- 0.2 (jitter = 1.1) 8.052 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.001 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.004 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.002 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.073 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.032 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.022 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.037 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.008 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.082 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.005 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.012 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 7.983 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.068 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.061 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 7.991 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 8.062 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 8.000 30 images/sec: 187.8 +/- 0.2 (jitter = 0.9) 7.973 30 images/sec: 187.8 +/- 0.2 (jitter = 1.1) 7.997 30 images/sec: 187.8 +/- 0.2 (jitter = 1.1) 8.010 30 images/sec: 187.8 +/- 0.2 (jitter = 0.8) 8.035 30 images/sec: 187.8 +/- 0.2 (jitter = 1.0) 10.328 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.990 40 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.981 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.986 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 8.002 40 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.987 40 images/sec: 188.0 +/- 0.1 (jitter = 0.7) 7.995 40 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.990 40 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 7.995 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.944 40 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 8.003 40 images/sec: 188.0 +/- 0.1 (jitter = 0.7) 7.981 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 8.001 40 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 8.006 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.989 40 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 8.207 40 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.951 40 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 7.973 40 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 7.985 40 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.989 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 8.053 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 9.882 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.979 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.980 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.978 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.983 40 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 8.076 40 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.987 40 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 7.967 40 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 8.153 40 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 8.077 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.983 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.990 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 8.004 40 images/sec: 188.0 +/- 0.2 (jitter = 0.9) 8.023 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.986 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.987 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.973 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.997 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.998 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.967 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.983 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 8.030 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.972 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.994 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.971 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.982 40 images/sec: 188.0 +/- 0.2 (jitter = 0.9) 7.992 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.957 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.996 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.983 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.993 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.995 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 8.002 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.990 40 images/sec: 188.0 +/- 0.2 (jitter = 0.9) 7.995 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.971 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.985 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 8.006 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.977 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 8.009 40 images/sec: 188.0 +/- 0.2 (jitter = 1.1) 7.980 40 images/sec: 188.0 +/- 0.2 (jitter = 0.8) 7.999 40 images/sec: 188.0 +/- 0.2 (jitter = 1.2) 8.015 40 images/sec: 188.0 +/- 0.2 (jitter = 1.0) 7.986 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.994 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 7.989 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.988 50 images/sec: 188.2 +/- 0.1 (jitter = 1.1) 7.985 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 7.989 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.989 50 images/sec: 188.2 +/- 0.1 (jitter = 0.9) 7.988 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.982 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.974 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.980 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.993 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 7.963 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.973 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 8.002 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 8.017 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 8.134 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 8.006 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.979 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.979 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 8.700 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 8.003 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.976 50 images/sec: 188.1 +/- 0.1 (jitter = 1.2) 7.989 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.975 50 images/sec: 188.1 +/- 0.1 (jitter = 1.2) 7.977 50 images/sec: 188.1 +/- 0.1 (jitter = 1.2) 7.983 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.981 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.991 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 8.058 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 8.018 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.984 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 7.994 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.970 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.972 50 images/sec: 188.1 +/- 0.1 (jitter = 1.2) 7.977 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 8.002 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.996 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.984 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.985 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.986 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 8.012 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.982 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.988 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.997 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 8.000 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.977 50 images/sec: 188.1 +/- 0.2 (jitter = 1.1) 7.975 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.996 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.983 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.993 50 images/sec: 188.1 +/- 0.2 (jitter = 0.9) 7.962 50 images/sec: 188.1 +/- 0.2 (jitter = 1.2) 7.977 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.984 50 images/sec: 188.1 +/- 0.1 (jitter = 1.2) 8.105 50 images/sec: 188.1 +/- 0.1 (jitter = 0.9) 7.982 50 images/sec: 188.1 +/- 0.1 (jitter = 1.2) 7.970 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.976 50 images/sec: 188.1 +/- 0.2 (jitter = 1.1) 8.043 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.982 50 images/sec: 188.1 +/- 0.2 (jitter = 1.1) 7.963 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 8.005 50 images/sec: 188.1 +/- 0.2 (jitter = 1.1) 8.027 50 images/sec: 188.1 +/- 0.1 (jitter = 1.0) 7.996 50 images/sec: 188.1 +/- 0.1 (jitter = 1.1) 7.962 60 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.996 60 images/sec: 188.1 +/- 0.1 (jitter = 1.2) 7.967 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.980 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.979 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.991 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.960 60 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.976 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.985 60 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.984 60 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 7.977 60 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.978 60 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.986 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.962 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.975 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.988 60 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.980 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.997 60 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.983 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 8.193 60 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.990 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.988 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.993 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.988 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.969 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.967 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.984 60 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.976 60 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.985 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.979 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.986 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.969 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.983 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.992 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.986 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.984 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.978 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.975 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.976 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.969 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.968 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.964 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.981 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.975 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.974 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.972 60 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.990 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.960 60 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.987 60 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.979 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.988 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.980 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.971 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.985 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.976 60 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.970 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.990 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 8.015 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.972 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.983 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.985 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.968 60 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.981 60 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 8.150 60 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.979 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.958 70 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 8.001 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.980 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.979 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.970 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.971 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.980 70 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.976 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.961 70 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.976 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 8.376 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.974 70 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.966 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.976 70 images/sec: 188.0 +/- 0.1 (jitter = 0.8) 7.978 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.985 70 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.990 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.991 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.975 70 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.963 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.959 70 images/sec: 188.0 +/- 0.1 (jitter = 0.9) 7.998 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.973 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.978 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.988 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.977 70 images/sec: 188.0 +/- 0.1 (jitter = 1.0) 7.978 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.971 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.976 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.984 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.974 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.985 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.979 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.974 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.997 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.978 70 images/sec: 188.0 +/- 0.1 (jitter = 1.4) 7.985 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.977 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.976 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.976 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.973 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.982 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.978 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.982 70 images/sec: 188.0 +/- 0.1 (jitter = 1.4) 7.972 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.974 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.979 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.966 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.977 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.984 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.980 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.973 70 images/sec: 188.0 +/- 0.1 (jitter = 1.3) 7.965 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.972 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.972 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.973 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.973 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.981 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.968 70 images/sec: 188.0 +/- 0.1 (jitter = 1.4) 7.970 70 images/sec: 188.0 +/- 0.1 (jitter = 1.2) 7.970 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.983 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.980 70 images/sec: 188.0 +/- 0.1 (jitter = 1.1) 7.968 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.968 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.957 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.976 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.970 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.961 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.971 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.963 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.964 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.971 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.971 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.961 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.964 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.974 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.966 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.974 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.968 80 images/sec: 187.9 +/- 0.1 (jitter = 0.9) 7.950 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.977 80 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.965 80 images/sec: 187.9 +/- 0.1 (jitter = 0.8) 7.959 80 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.975 80 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.963 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.970 80 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.979 80 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.970 80 images/sec: 187.9 +/- 0.1 (jitter = 0.9) 7.974 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.966 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.965 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.962 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.978 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.960 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.959 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.970 80 images/sec: 187.9 +/- 0.1 (jitter = 1.5) 7.966 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 8.001 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.963 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.968 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.973 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.978 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.974 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.955 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.971 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.971 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.958 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.953 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.963 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.968 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.964 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.971 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.971 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.972 80 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.965 80 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.983 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.966 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.965 80 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.973 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.952 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.955 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.955 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.961 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.968 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.962 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.953 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.955 90 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.953 90 images/sec: 187.9 +/- 0.1 (jitter = 1.1) 7.963 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.970 90 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.950 90 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.962 90 images/sec: 187.9 +/- 0.1 (jitter = 1.1) 7.976 90 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.965 90 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.960 90 images/sec: 187.9 +/- 0.1 (jitter = 1.1) 7.962 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.972 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.968 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.959 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.957 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.955 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.970 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.948 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.957 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.950 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.963 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.963 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.959 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.962 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.962 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.958 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.956 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.948 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.962 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.958 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.958 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.944 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.963 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.967 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.970 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.959 90 images/sec: 187.9 +/- 0.1 (jitter = 1.5) 7.953 90 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.969 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.958 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.960 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.956 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.959 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.965 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.948 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.957 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.960 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.966 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.962 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.970 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.970 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.952 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.957 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.959 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.964 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.954 90 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.956 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.958 90 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.954 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.958 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.953 ---------------------------------------------------------------- total images/sec: 12025.08 ---------------------------------------------------------------- total images/sec: 12025.22 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.957 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.954 ---------------------------------------------------------------- total images/sec: 12025.18 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.945 ---------------------------------------------------------------- total images/sec: 12025.19 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.18 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.975 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.957 ---------------------------------------------------------------- total images/sec: 12025.39 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.14 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.949 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.949 ---------------------------------------------------------------- total images/sec: 12025.29 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.21 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.958 ---------------------------------------------------------------- total images/sec: 12025.24 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.953 ---------------------------------------------------------------- total images/sec: 12025.18 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.961 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.945 total images/sec: 12025.29 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.15 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.952 ---------------------------------------------------------------- total images/sec: 12025.24 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.958 ---------------------------------------------------------------- total images/sec: 12025.14 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.949 ---------------------------------------------------------------- total images/sec: 12025.21 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.955 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.961 ---------------------------------------------------------------- total images/sec: 12025.11 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.944 ---------------------------------------------------------------- total images/sec: 12025.17 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.14 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.954 ---------------------------------------------------------------- total images/sec: 12025.19 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.952 ---------------------------------------------------------------- total images/sec: 12025.22 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.943 ---------------------------------------------------------------- total images/sec: 12025.17 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.955 ---------------------------------------------------------------- total images/sec: 12025.38 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.953 ---------------------------------------------------------------- total images/sec: 12025.24 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.945 100 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.969 ---------------------------------------------------------------- total images/sec: 12025.22 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 0.9) 7.951 ---------------------------------------------------------------- total images/sec: 12024.73 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.952 100 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.958 ---------------------------------------------------------------- total images/sec: 12024.67 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.14 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12024.70 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.974 ---------------------------------------------------------------- total images/sec: 12024.69 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.951 ---------------------------------------------------------------- total images/sec: 12024.60 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.966 ---------------------------------------------------------------- total images/sec: 12024.49 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.958 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.963 ---------------------------------------------------------------- total images/sec: 12024.52 ---------------------------------------------------------------- total images/sec: 12025.22 ---------------------------------------------------------------- ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.0) 7.950 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.960 ---------------------------------------------------------------- total images/sec: 12024.42 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.14 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.960 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.951 ---------------------------------------------------------------- total images/sec: 12025.28 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.10 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.950 ---------------------------------------------------------------- total images/sec: 12025.11 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.5) 7.946 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.947 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.954 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.978 ---------------------------------------------------------------- total images/sec: 12025.12 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.954 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.961 ---------------------------------------------------------------- total images/sec: 12025.13 ---------------------------------------------------------------- total images/sec: 12025.33 ---------------------------------------------------------------- total images/sec: 12025.07 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.954 ---------------------------------------------------------------- total images/sec: 12025.10 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.33 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.13 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.949 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.945 ---------------------------------------------------------------- total images/sec: 12025.23 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.14 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.941 ---------------------------------------------------------------- total images/sec: 12025.19 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.955 ---------------------------------------------------------------- total images/sec: 12025.29 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 8.105 ---------------------------------------------------------------- total images/sec: 12025.22 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.960 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.952 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.948 ---------------------------------------------------------------- total images/sec: 12025.13 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.08 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.954 ---------------------------------------------------------------- total images/sec: 12025.15 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.954 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.955 ---------------------------------------------------------------- total images/sec: 12025.19 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.21 ---------------------------------------------------------------- total images/sec: 12025.16 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.954 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.955 ---------------------------------------------------------------- total images/sec: 12025.12 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.19 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.956 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.961 100 images/sec: 187.9 +/- 0.1 (jitter = 1.4) 7.946 ---------------------------------------------------------------- total images/sec: 12025.18 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.08 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.24 ---------------------------------------------------------------- 100 images/sec: 187.9 +/- 0.1 (jitter = 1.3) 7.944 100 images/sec: 187.9 +/- 0.1 (jitter = 1.2) 7.952 ---------------------------------------------------------------- total images/sec: 12025.10 ---------------------------------------------------------------- ---------------------------------------------------------------- total images/sec: 12025.17 ----------------------------------------------------------------