+ POD_NAME=nccl-test-worker-0 + shift + /opt/kube/kubectl exec nccl-test-worker-0 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 1 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=nccl-test-worker-2 + shift + /opt/kube/kubectl exec nccl-test-worker-2 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 3 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=nccl-test-worker-1 + shift + + /opt/kube/kubectl execPOD_NAME=nccl-test-worker-5 nccl-test-worker-1 --+ /bin/sh -cshift orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 2 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" + /opt/kube/kubectl exec nccl-test-worker-5 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 6 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=nccl-test-worker-3 + shift + /opt/kube/kubectl exec nccl-test-worker-3 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 4 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" + + POD_NAME=nccl-test-worker-4POD_NAME=nccl-test-worker-6 + + shiftshift + + /opt/kube/kubectl exec/opt/kube/kubectl nccl-test-worker-6 exec -- nccl-test-worker-4 /bin/sh -- -c /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 7 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 5 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" + POD_NAME=nccl-test-worker-7 + shift + /opt/kube/kubectl exec nccl-test-worker-7 -- /bin/sh -c orted -mca ess "env" -mca ess_base_jobid "1152057344" -mca ess_base_vpid 8 -mca ess_base_num_procs "9" -mca orte_node_regex "nccl-test-launcher-rcql[1:2],nccl-test-worker-[1:0-7]@0(9)" -mca orte_hnp_uri "1152057344.0;tcp://192.168.49.108:54725" --mca plm_rsh_no_tree_spawn "1" --mca pml "ob1" --mca btl_vader_single_copy_mechanism "none" --mca btl_tcp_if_exclude "lo,docker0" -mca plm "rsh" -mca orte_default_hostfile "/etc/mpi/hostfile" -mca plm_rsh_agent "/etc/mpi/kubexec.sh" -mca hwloc_base_binding_policy "none" -mca rmaps_base_oversubscribe "1" -mca pmix "^s1,s2,cray,isolated" # nThread 1 nGpus 1 minBytes 8 maxBytes 1073741824 step: 2(factor) warmup iters: 5 iters: 500 validation: 1 # # Using devices # Rank 0 Pid 14 on nccl-test-worker-0 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 1 Pid 15 on nccl-test-worker-0 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 2 Pid 16 on nccl-test-worker-0 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 3 Pid 17 on nccl-test-worker-0 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 4 Pid 18 on nccl-test-worker-0 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 5 Pid 19 on nccl-test-worker-0 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 6 Pid 20 on nccl-test-worker-0 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 7 Pid 21 on nccl-test-worker-0 device 7 [0x00] Tesla V100-SXM2-32GB # Rank 8 Pid 14 on nccl-test-worker-1 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 9 Pid 15 on nccl-test-worker-1 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 10 Pid 16 on nccl-test-worker-1 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 11 Pid 17 on nccl-test-worker-1 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 12 Pid 18 on nccl-test-worker-1 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 13 Pid 19 on nccl-test-worker-1 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 14 Pid 25 on nccl-test-worker-1 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 15 Pid 30 on nccl-test-worker-1 device 7 [0x00] Tesla V100-SXM2-32GB # Rank 16 Pid 14 on nccl-test-worker-2 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 17 Pid 15 on nccl-test-worker-2 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 18 Pid 16 on nccl-test-worker-2 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 19 Pid 17 on nccl-test-worker-2 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 20 Pid 18 on nccl-test-worker-2 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 21 Pid 19 on nccl-test-worker-2 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 22 Pid 20 on nccl-test-worker-2 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 23 Pid 21 on nccl-test-worker-2 device 7 [0x00] Tesla V100-SXM2-32GB # Rank 24 Pid 14 on nccl-test-worker-3 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 25 Pid 15 on nccl-test-worker-3 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 26 Pid 16 on nccl-test-worker-3 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 27 Pid 17 on nccl-test-worker-3 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 28 Pid 18 on nccl-test-worker-3 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 29 Pid 19 on nccl-test-worker-3 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 30 Pid 21 on nccl-test-worker-3 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 31 Pid 24 on nccl-test-worker-3 device 7 [0x00] Tesla V100-SXM2-32GB # Rank 32 Pid 14 on nccl-test-worker-4 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 33 Pid 15 on nccl-test-worker-4 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 34 Pid 17 on nccl-test-worker-4 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 35 Pid 19 on nccl-test-worker-4 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 36 Pid 20 on nccl-test-worker-4 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 37 Pid 22 on nccl-test-worker-4 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 38 Pid 24 on nccl-test-worker-4 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 39 Pid 25 on nccl-test-worker-4 device 7 [0x00] Tesla V100-SXM2-32GB # Rank 40 Pid 14 on nccl-test-worker-5 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 41 Pid 15 on nccl-test-worker-5 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 42 Pid 16 on nccl-test-worker-5 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 43 Pid 17 on nccl-test-worker-5 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 44 Pid 18 on nccl-test-worker-5 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 45 Pid 19 on nccl-test-worker-5 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 46 Pid 20 on nccl-test-worker-5 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 47 Pid 25 on nccl-test-worker-5 device 7 [0x00] Tesla V100-SXM2-32GB # Rank 48 Pid 14 on nccl-test-worker-6 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 49 Pid 15 on nccl-test-worker-6 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 50 Pid 16 on nccl-test-worker-6 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 51 Pid 18 on nccl-test-worker-6 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 52 Pid 20 on nccl-test-worker-6 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 53 Pid 22 on nccl-test-worker-6 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 54 Pid 24 on nccl-test-worker-6 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 55 Pid 25 on nccl-test-worker-6 device 7 [0x00] Tesla V100-SXM2-32GB # Rank 56 Pid 14 on nccl-test-worker-7 device 0 [0x00] Tesla V100-SXM2-32GB # Rank 57 Pid 15 on nccl-test-worker-7 device 1 [0x00] Tesla V100-SXM2-32GB # Rank 58 Pid 16 on nccl-test-worker-7 device 2 [0x00] Tesla V100-SXM2-32GB # Rank 59 Pid 17 on nccl-test-worker-7 device 3 [0x00] Tesla V100-SXM2-32GB # Rank 60 Pid 20 on nccl-test-worker-7 device 4 [0x00] Tesla V100-SXM2-32GB # Rank 61 Pid 22 on nccl-test-worker-7 device 5 [0x00] Tesla V100-SXM2-32GB # Rank 62 Pid 24 on nccl-test-worker-7 device 6 [0x00] Tesla V100-SXM2-32GB # Rank 63 Pid 25 on nccl-test-worker-7 device 7 [0x00] Tesla V100-SXM2-32GB nccl-test-worker-0:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:16:16 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:16:16 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:16:16 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:16:16 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:16:16 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:16:16 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:22:22 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:22:22 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:22:22 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:22:22 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:22:22 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:22:22 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:18:18 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:25:25 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:18:18 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:25:25 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:18:18 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:18:18 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:25:25 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:25:25 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:25:25 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:25:25 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:18:18 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:18:18 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:24:24 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:24:24 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:24:24 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:24:24 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:24:24 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:24:24 [6] NCCL INFO Using network AWS Libfabric nccl-test-worker-6:20:20 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.50.4<0> nccl-test-worker-6:20:20 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-6:20:20 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-6:20:20 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-6:20:20 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-6:20:20 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:25:25 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:25:25 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:25:25 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:25:25 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:25:25 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:25:25 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:24:24 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:24:24 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:24:24 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:24:24 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:24:24 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:24:24 [6] NCCL INFO Using network AWS Libfabric NCCL version 2.6.4+cuda10.2 nccl-test-worker-2:16:16 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:16:16 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:16:16 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:16:16 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:16:16 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:16:16 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:16:16 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:17:17 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:20:20 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:22:22 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.58.173<0> nccl-test-worker-7:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:16:16 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:17:17 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:20:20 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:22:22 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-7:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:20:20 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:20:20 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:20:20 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:20:20 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:22:22 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:22:22 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:22:22 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:22:22 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:16:16 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:16:16 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:16:16 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:16:16 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-7:17:17 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-7:17:17 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-7:17:17 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-7:17:17 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:17:17 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:18:18 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:19:19 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:20:20 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:21:21 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.60.91<0> nccl-test-worker-2:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:17:17 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:19:19 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:20:20 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:21:21 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-2:18:18 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-2:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-2:20:20 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:20:20 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:20:20 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:20:20 [6] NCCL INFO Using network AWS Libfabric nccl-test-worker-2:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-2:18:18 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:18:18 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:18:18 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:18:18 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-2:21:21 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:21:21 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:21:21 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:21:21 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-2:17:17 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:17:17 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:17:17 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:17:17 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-2:19:19 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-2:19:19 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-2:19:19 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-2:19:19 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:30:30 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:30:30 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:30:30 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:30:30 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:30:30 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:30:30 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:25:25 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:25:25 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:25:25 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:25:25 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:25:25 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:25:25 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:16:16 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:18:18 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:19:19 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:16:16 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:18:18 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:19:19 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:18:18 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:18:18 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:18:18 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:18:18 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:16:16 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:16:16 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:16:16 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:16:16 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:19:19 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:19:19 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:19:19 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:19:19 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:17:17 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:25:25 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.57.6<0> nccl-test-worker-1:17:17 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:25:25 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-1:17:17 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:17:17 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:17:17 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:17:17 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-1:25:25 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-1:25:25 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-1:25:25 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-1:25:25 [6] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:17:17 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:19:19 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:20:20 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:17:17 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:19:19 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:20:20 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:19:19 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:19:19 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:19:19 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:19:19 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:17:17 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:17:17 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:17:17 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:17:17 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:20:20 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:20:20 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:20:20 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:20:20 [6] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:16:16 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:18:18 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.34.178<0> nccl-test-worker-5:16:16 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:18:18 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-5:16:16 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:16:16 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:16:16 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:16:16 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-5:18:18 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-5:18:18 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-5:18:18 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-5:18:18 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:22:22 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:22:22 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:22:22 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:22:22 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:22:22 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:22:22 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:19:19 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:19:19 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:19:19 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:19:19 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:19:19 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:19:19 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:17:17 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:20:20 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:24:24 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:25:25 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.55.111<0> nccl-test-worker-4:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:17:17 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:20:20 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:24:24 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:25:25 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-4:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:24:24 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:24:24 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:24:24 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:24:24 [6] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:25:25 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:25:25 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:25:25 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:25:25 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:17:17 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:17:17 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:17:17 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:17:17 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-4:20:20 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-4:20:20 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-4:20:20 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-4:20:20 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:18:18 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:18:18 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:18:18 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:18:18 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:18:18 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:18:18 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:19:19 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:20:20 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:19:19 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:20:20 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:19:19 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:19:19 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:19:19 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:19:19 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:20:20 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:20:20 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:20:20 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:20:20 [6] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:16:16 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:17:17 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:21:21 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.36.141<0> nccl-test-worker-0:16:16 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:17:17 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:21:21 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-0:17:17 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:17:17 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:17:17 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:17:17 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:16:16 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:16:16 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:16:16 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:16:16 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:21:21 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-0:21:21 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-0:21:21 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-0:21:21 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:15:15 [1] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:16:16 [2] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:18:18 [4] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:21:21 [6] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:15:15 [1] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:16:16 [2] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:18:18 [4] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:21:21 [6] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:18:18 [4] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:18:18 [4] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:18:18 [4] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:18:18 [4] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:21:21 [6] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:21:21 [6] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:21:21 [6] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:21:21 [6] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:15:15 [1] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:15:15 [1] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:15:15 [1] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:15:15 [1] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:16:16 [2] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:16:16 [2] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:16:16 [2] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:16:16 [2] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:24:24 [7] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:24:24 [7] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:24:24 [7] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:24:24 [7] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:24:24 [7] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:24:24 [7] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:14:14 [0] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:14:14 [0] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:14:14 [0] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:14:14 [0] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:14:14 [0] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:14:14 [0] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:19:19 [5] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:17:17 [3] NCCL INFO Bootstrap : Using [0]eth0:192.168.56.162<0> nccl-test-worker-3:19:19 [5] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:17:17 [3] NCCL INFO NET/OFI Setting RDMAV_FORK_SAFE environment variable to 1. nccl-test-worker-3:19:19 [5] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:19:19 [5] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:19:19 [5] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:19:19 [5] NCCL INFO Using network AWS Libfabric nccl-test-worker-3:17:17 [3] NCCL INFO NET/OFI Forcing AWS OFI ndev 4 nccl-test-worker-3:17:17 [3] NCCL INFO NET/OFI Selected Provider is efa nccl-test-worker-3:17:17 [3] NCCL INFO NET/Plugin: Failed to find ncclCollNetPlugin_v3 symbol. nccl-test-worker-3:17:17 [3] NCCL INFO Using network AWS Libfabric nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:25:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:20:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:14:57 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:22:54 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:17:61 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:17:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:21:61 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:21:54 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:14:61 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:24:56 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:14:54 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-3:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:18:58 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-3:14:59 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:19:55 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:20:58 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-4:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:17:62 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:15:60 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:16:54 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:19:57 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:19:61 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:17:59 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:16:58 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:20:59 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:22:55 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:22:59 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:16:61 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:25:59 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:17:57 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:15:58 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:16:60 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:20:57 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:18:55 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:24:55 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-7:14:56 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-0:18:56 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-2:14:58 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-0:14:55 [0] NCCL INFO NET/OFI [0] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-2:21:60 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:20:61 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:15:56 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-6:24:60 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:15:57 [1] NCCL INFO NET/OFI [1] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:18:60 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:17:56 [3] NCCL INFO NET/OFI [3] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:25:61 [6] NCCL INFO NET/OFI [6] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-1:16:57 [2] NCCL INFO NET/OFI [2] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:25:55 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-1:19:56 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00 nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-1:30:54 [7] NCCL INFO NET/OFI [7] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:18:59 [4] NCCL INFO NET/OFI [4] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 0 busId 0000:00:16.0 path /sys/devices/pci0000:00 nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 1 busId 0000:00:17.0 path /sys/devices/pci0000:00 nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 2 busId 0000:00:18.0 path /sys/devices/pci0000:00/ nccl-test-worker-5:19:60 [5] NCCL INFO NET/OFI [5] getCudaPath dev 3 busId 0000:00:19.0 path /sys/devices/pci0000:00 nccl-test-worker-5:19:60 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:19:60 [5] NCCL INFO Trees [0] 46/-1/-1->45->41|41->45->46/-1/-1 [1] -1/-1/-1->45->46|46->45->-1/-1/-1 [2] 41/-1/-1->45->46|46->45->41/-1/-1 [3] 46/-1/-1->45->54|54->45->46/-1/-1 [4] 46/-1/-1->45->41|41->45->46/-1/-1 [5] -1/-1/-1->45->46|46->45->-1/-1/-1 [6] 41/-1/-1->45->46|46->45->41/-1/-1 [7] 46/-1/-1->45->14|14->45->46/-1/-1 nccl-test-worker-5:20:58 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:20:58 [6] NCCL INFO Trees [0] 47/-1/-1->46->45|45->46->47/-1/-1 [1] 45/-1/-1->46->47|47->46->45/-1/-1 [2] 45/-1/-1->46->47|47->46->45/-1/-1 [3] 47/-1/-1->46->45|45->46->47/-1/-1 [4] 47/-1/-1->46->45|45->46->47/-1/-1 [5] 45/-1/-1->46->47|47->46->45/-1/-1 [6] 45/-1/-1->46->47|47->46->45/-1/-1 [7] 47/29/61->46->45|45->46->47/29/61 nccl-test-worker-6:18:58 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:18:58 [3] NCCL INFO Trees [0] 50/40/56->51->48|48->51->50/40/56 [1] 48/-1/-1->51->50|50->51->48/-1/-1 [2] 48/-1/-1->51->50|50->51->48/-1/-1 [3] 50/-1/-1->51->48|48->51->50/-1/-1 [4] 50/-1/-1->51->48|48->51->50/-1/-1 [5] 48/-1/-1->51->50|50->51->48/-1/-1 [6] 48/-1/-1->51->50|50->51->48/-1/-1 [7] 50/-1/-1->51->48|48->51->50/-1/-1 nccl-test-worker-6:25:59 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:25:59 [7] NCCL INFO Trees [0] 52/-1/-1->55->54|54->55->52/-1/-1 [1] 54/-1/-1->55->52|52->55->54/-1/-1 [2] 54/44/60->55->52|52->55->54/44/60 [3] 52/-1/-1->55->54|54->55->52/-1/-1 [4] 52/-1/-1->55->54|54->55->52/-1/-1 [5] 54/-1/-1->55->52|52->55->54/-1/-1 [6] 54/-1/-1->55->52|52->55->54/-1/-1 [7] 52/-1/-1->55->54|54->55->52/-1/-1 nccl-test-worker-6:14:57 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:14:57 [0] NCCL INFO Trees [0] 51/-1/-1->48->35|35->48->51/-1/-1 [1] 52/-1/-1->48->51|51->48->52/-1/-1 [2] -1/-1/-1->48->51|51->48->-1/-1/-1 [3] 51/-1/-1->48->52|52->48->51/-1/-1 [4] 51/-1/-1->48->59|59->48->51/-1/-1 [5] 52/-1/-1->48->51|51->48->52/-1/-1 [6] -1/-1/-1->48->51|51->48->-1/-1/-1 [7] 51/-1/-1->48->52|52->48->51/-1/-1 nccl-test-worker-6:16:54 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:16:54 [2] NCCL INFO Trees [0] 49/-1/-1->50->51|51->50->49/-1/-1 [1] 51/41/57->50->49|49->50->51/41/57 [2] 51/-1/-1->50->49|49->50->51/-1/-1 [3] 49/-1/-1->50->51|51->50->49/-1/-1 [4] 49/-1/-1->50->51|51->50->49/-1/-1 [5] 51/-1/-1->50->49|49->50->51/-1/-1 [6] 51/-1/-1->50->49|49->50->51/-1/-1 [7] 49/-1/-1->50->51|51->50->49/-1/-1 nccl-test-worker-6:20:61 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:20:61 [4] NCCL INFO Trees [0] -1/-1/-1->52->55|55->52->-1/-1/-1 [1] 55/-1/-1->52->48|48->52->55/-1/-1 [2] 55/-1/-1->52->39|39->52->55/-1/-1 [3] 48/-1/-1->52->55|55->52->48/-1/-1 [4] -1/-1/-1->52->55|55->52->-1/-1/-1 [5] 55/-1/-1->52->48|48->52->55/-1/-1 [6] 55/-1/-1->52->63|63->52->55/-1/-1 [7] 48/-1/-1->52->55|55->52->48/-1/-1 nccl-test-worker-6:15:56 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:15:56 [1] NCCL INFO Trees [0] 53/-1/-1->49->50|50->49->53/-1/-1 [1] 50/-1/-1->49->34|34->49->50/-1/-1 [2] 50/-1/-1->49->53|53->49->50/-1/-1 [3] -1/-1/-1->49->50|50->49->-1/-1/-1 [4] 53/-1/-1->49->50|50->49->53/-1/-1 [5] 50/-1/-1->49->58|58->49->50/-1/-1 [6] 50/-1/-1->49->53|53->49->50/-1/-1 [7] -1/-1/-1->49->50|50->49->-1/-1/-1 nccl-test-worker-6:22:55 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:22:55 [5] NCCL INFO Trees [0] 54/-1/-1->53->49|49->53->54/-1/-1 [1] -1/-1/-1->53->54|54->53->-1/-1/-1 [2] 49/-1/-1->53->54|54->53->49/-1/-1 [3] 54/-1/-1->53->38|38->53->54/-1/-1 [4] 54/-1/-1->53->49|49->53->54/-1/-1 [5] -1/-1/-1->53->54|54->53->-1/-1/-1 [6] 49/-1/-1->53->54|54->53->49/-1/-1 [7] 54/-1/-1->53->62|62->53->54/-1/-1 nccl-test-worker-6:24:60 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-6:24:60 [6] NCCL INFO Trees [0] 55/-1/-1->54->53|53->54->55/-1/-1 [1] 53/-1/-1->54->55|55->54->53/-1/-1 [2] 53/-1/-1->54->55|55->54->53/-1/-1 [3] 55/45/61->54->53|53->54->55/45/61 [4] 55/-1/-1->54->53|53->54->55/-1/-1 [5] 53/-1/-1->54->55|55->54->53/-1/-1 [6] 53/-1/-1->54->55|55->54->53/-1/-1 [7] 55/-1/-1->54->53|53->54->55/-1/-1 nccl-test-worker-5:25:55 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:25:55 [7] NCCL INFO Trees [0] 44/-1/-1->47->46|46->47->44/-1/-1 [1] 46/-1/-1->47->44|44->47->46/-1/-1 [2] 46/-1/-1->47->44|44->47->46/-1/-1 [3] 44/-1/-1->47->46|46->47->44/-1/-1 [4] 44/-1/-1->47->46|46->47->44/-1/-1 [5] 46/-1/-1->47->44|44->47->46/-1/-1 [6] 46/28/60->47->44|44->47->46/28/60 [7] 44/-1/-1->47->46|46->47->44/-1/-1 nccl-test-worker-1:19:56 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-1:19:56 [5] NCCL INFO Trees [0] 14/-1/-1->13->9|9->13->14/-1/-1 [1] -1/-1/-1->13->14|14->13->-1/-1/-1 [2] 9/-1/-1->13->14|14->13->9/-1/-1 [3] 14/-1/-1->13->22|22->13->14/-1/-1 [4] 14/-1/-1->13->9|9->13->14/-1/-1 [5] -1/-1/-1->13->14|14->13->-1/-1/-1 [6] 9/-1/-1->13->14|14->13->9/-1/-1 [7] 14/-1/-1->13->-1|-1->13->14/-1/-1 nccl-test-worker-1:17:59 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-1:17:59 [3] NCCL INFO Trees [0] 10/-1/-1->11->8|8->11->10/-1/-1 [1] 8/-1/-1->11->10|10->11->8/-1/-1 [2] 8/-1/-1->11->10|10->11->8/-1/-1 [3] 10/-1/-1->11->8|8->11->10/-1/-1 [4] 10/40/-1->11->8|8->11->10/40/-1 [5] 8/-1/-1->11->10|10->11->8/-1/-1 [6] 8/-1/-1->11->10|10->11->8/-1/-1 [7] 10/-1/-1->11->8|8->11->10/-1/-1 nccl-test-worker-0:20:59 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:20:59 [6] NCCL INFO Trees [0] 7/-1/-1->6->5|5->6->7/-1/-1 [1] 5/-1/-1->6->7|7->6->5/-1/-1 [2] 5/-1/-1->6->7|7->6->5/-1/-1 [3] 7/37/-1->6->5|5->6->7/37/-1 [4] 7/-1/-1->6->5|5->6->7/-1/-1 [5] 5/-1/-1->6->7|7->6->5/-1/-1 [6] 5/-1/-1->6->7|7->6->5/-1/-1 [7] 7/-1/-1->6->5|5->6->7/-1/-1 nccl-test-worker-1:25:61 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-1:25:61 [6] NCCL INFO Trees [0] 15/-1/-1->14->13|13->14->15/-1/-1 [1] 13/-1/-1->14->15|15->14->13/-1/-1 [2] 13/-1/-1->14->15|15->14->13/-1/-1 [3] 15/-1/-1->14->13|13->14->15/-1/-1 [4] 15/-1/-1->14->13|13->14->15/-1/-1 [5] 13/-1/-1->14->15|15->14->13/-1/-1 [6] 13/-1/-1->14->15|15->14->13/-1/-1 [7] 15/45/-1->14->13|13->14->15/45/-1 nccl-test-worker-7:15:60 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:15:60 [1] NCCL INFO Trees [0] 61/-1/-1->57->58|58->57->61/-1/-1 [1] 58/-1/-1->57->50|50->57->58/-1/-1 [2] 58/-1/-1->57->61|61->57->58/-1/-1 [3] -1/-1/-1->57->58|58->57->-1/-1/-1 [4] 61/-1/-1->57->58|58->57->61/-1/-1 [5] 58/-1/-1->57->42|42->57->58/-1/-1 [6] 58/-1/-1->57->61|61->57->58/-1/-1 [7] -1/-1/-1->57->58|58->57->-1/-1/-1 nccl-test-worker-0:19:57 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:19:57 [5] NCCL INFO Trees [0] 6/-1/-1->5->1|1->5->6/-1/-1 [1] -1/-1/-1->5->6|6->5->-1/-1/-1 [2] 1/-1/-1->5->6|6->5->1/-1/-1 [3] 6/-1/-1->5->-1|-1->5->6/-1/-1 [4] 6/-1/-1->5->1|1->5->6/-1/-1 [5] -1/-1/-1->5->6|6->5->-1/-1/-1 [6] 1/-1/-1->5->6|6->5->1/-1/-1 [7] 6/-1/-1->5->62|62->5->6/-1/-1 nccl-test-worker-2:14:58 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:14:58 [0] NCCL INFO Trees [0] 19/-1/-1->16->35|35->16->19/-1/-1 [1] 20/-1/-1->16->19|19->16->20/-1/-1 [2] -1/-1/-1->16->19|19->16->-1/-1/-1 [3] 19/-1/-1->16->20|20->16->19/-1/-1 [4] 19/-1/-1->16->27|27->16->19/-1/-1 [5] 20/-1/-1->16->19|19->16->20/-1/-1 [6] -1/-1/-1->16->19|19->16->-1/-1/-1 [7] 19/-1/-1->16->20|20->16->19/-1/-1 nccl-test-worker-1:18:60 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-1:18:60 [4] NCCL INFO Trees [0] -1/-1/-1->12->15|15->12->-1/-1/-1 [1] 15/-1/-1->12->8|8->12->15/-1/-1 [2] 15/-1/-1->12->23|23->12->15/-1/-1 [3] 8/-1/-1->12->15|15->12->8/-1/-1 [4] -1/-1/-1->12->15|15->12->-1/-1/-1 [5] 15/-1/-1->12->8|8->12->15/-1/-1 [6] 15/-1/-1->12->-1|-1->12->15/-1/-1 [7] 8/-1/-1->12->15|15->12->8/-1/-1 nccl-test-worker-7:24:55 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:24:55 [6] NCCL INFO Trees [0] 63/-1/-1->62->61|61->62->63/-1/-1 [1] 61/-1/-1->62->63|63->62->61/-1/-1 [2] 61/-1/-1->62->63|63->62->61/-1/-1 [3] 63/-1/-1->62->61|61->62->63/-1/-1 [4] 63/-1/-1->62->61|61->62->63/-1/-1 [5] 61/-1/-1->62->63|63->62->61/-1/-1 [6] 61/-1/-1->62->63|63->62->61/-1/-1 [7] 63/53/5->62->61|61->62->63/53/5 nccl-test-worker-0:18:56 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:18:56 [4] NCCL INFO Trees [0] -1/-1/-1->4->7|7->4->-1/-1/-1 [1] 7/-1/-1->4->0|0->4->7/-1/-1 [2] 7/-1/-1->4->-1|-1->4->7/-1/-1 [3] 0/-1/-1->4->7|7->4->0/-1/-1 [4] -1/-1/-1->4->7|7->4->-1/-1/-1 [5] 7/-1/-1->4->0|0->4->7/-1/-1 [6] 7/-1/-1->4->63|63->4->7/-1/-1 [7] 0/-1/-1->4->7|7->4->0/-1/-1 nccl-test-worker-2:15:56 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:15:56 [1] NCCL INFO Trees [0] 21/-1/-1->17->18|18->17->21/-1/-1 [1] 18/-1/-1->17->34|34->17->18/-1/-1 [2] 18/-1/-1->17->21|21->17->18/-1/-1 [3] -1/-1/-1->17->18|18->17->-1/-1/-1 [4] 21/-1/-1->17->18|18->17->21/-1/-1 [5] 18/-1/-1->17->26|26->17->18/-1/-1 [6] 18/-1/-1->17->21|21->17->18/-1/-1 [7] -1/-1/-1->17->18|18->17->-1/-1/-1 nccl-test-worker-1:30:54 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-1:30:54 [7] NCCL INFO Trees [0] 12/-1/-1->15->14|14->15->12/-1/-1 [1] 14/-1/-1->15->12|12->15->14/-1/-1 [2] 14/-1/-1->15->12|12->15->14/-1/-1 [3] 12/-1/-1->15->14|14->15->12/-1/-1 [4] 12/-1/-1->15->14|14->15->12/-1/-1 [5] 14/-1/-1->15->12|12->15->14/-1/-1 [6] 14/44/-1->15->12|12->15->14/44/-1 [7] 12/-1/-1->15->14|14->15->12/-1/-1 nccl-test-worker-7:20:57 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:20:57 [4] NCCL INFO Trees [0] -1/-1/-1->60->63|63->60->-1/-1/-1 [1] 63/-1/-1->60->56|56->60->63/-1/-1 [2] 63/-1/-1->60->55|55->60->63/-1/-1 [3] 56/-1/-1->60->63|63->60->56/-1/-1 [4] -1/-1/-1->60->63|63->60->-1/-1/-1 [5] 63/-1/-1->60->56|56->60->63/-1/-1 [6] 63/-1/-1->60->47|47->60->63/-1/-1 [7] 56/-1/-1->60->63|63->60->56/-1/-1 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 00/08 : 0 3 2 1 5 6 7 4 8 11 10 9 13 14 15 12 16 19 18 17 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 01/08 : 0 4 7 6 5 9 10 11 8 12 15 14 13 17 18 19 16 20 23 22 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 02/08 : 0 1 5 6 10 11 15 12 8 9 13 14 18 19 23 20 16 17 21 22 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 03/08 : 0 1 2 6 5 4 7 11 8 9 10 14 13 12 15 19 16 17 18 22 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 04/08 : 0 3 2 1 5 6 7 4 8 11 10 9 13 14 15 12 16 19 18 17 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 05/08 : 0 4 7 6 5 9 10 11 8 12 15 14 13 17 18 19 16 20 23 22 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 06/08 : 0 1 5 6 10 11 15 12 8 9 13 14 18 19 23 20 16 17 21 22 nccl-test-worker-0:14:55 [0] NCCL INFO Channel 07/08 : 0 1 2 6 5 4 7 11 8 9 10 14 13 12 15 19 16 17 18 22 nccl-test-worker-2:16:54 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:16:54 [2] NCCL INFO Trees [0] 17/-1/-1->18->19|19->18->17/-1/-1 [1] 19/9/25->18->17|17->18->19/9/25 [2] 19/-1/-1->18->17|17->18->19/-1/-1 [3] 17/-1/-1->18->19|19->18->17/-1/-1 [4] 17/-1/-1->18->19|19->18->17/-1/-1 [5] 19/-1/-1->18->17|17->18->19/-1/-1 [6] 19/-1/-1->18->17|17->18->19/-1/-1 [7] 17/-1/-1->18->19|19->18->17/-1/-1 nccl-test-worker-1:16:57 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:22:59 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:22:59 [5] NCCL INFO Trees [0] 62/-1/-1->61->57|57->61->62/-1/-1 [1] -1/-1/-1->61->62|62->61->-1/-1/-1 [2] 57/-1/-1->61->62|62->61->57/-1/-1 [3] 62/-1/-1->61->54|54->61->62/-1/-1 [4] 62/-1/-1->61->57|57->61->62/-1/-1 [5] -1/-1/-1->61->62|62->61->-1/-1/-1 [6] 57/-1/-1->61->62|62->61->57/-1/-1 [7] 62/-1/-1->61->46|46->61->62/-1/-1 nccl-test-worker-0:15:58 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:15:58 [1] NCCL INFO Trees [0] 5/-1/-1->1->2|2->1->5/-1/-1 [1] 2/-1/-1->1->-1|-1->1->2/-1/-1 [2] 2/-1/-1->1->5|5->1->2/-1/-1 [3] -1/-1/-1->1->2|2->1->-1/-1/-1 [4] 5/-1/-1->1->2|2->1->5/-1/-1 [5] 2/-1/-1->1->58|58->1->2/-1/-1 [6] 2/-1/-1->1->5|5->1->2/-1/-1 [7] -1/-1/-1->1->2|2->1->-1/-1/-1 nccl-test-worker-2:17:57 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:17:57 [3] NCCL INFO Trees [0] 18/8/24->19->16|16->19->18/8/24 [1] 16/-1/-1->19->18|18->19->16/-1/-1 [2] 16/-1/-1->19->18|18->19->16/-1/-1 [3] 18/-1/-1->19->16|16->19->18/-1/-1 [4] 18/-1/-1->19->16|16->19->18/-1/-1 [5] 16/-1/-1->19->18|18->19->16/-1/-1 [6] 16/-1/-1->19->18|18->19->16/-1/-1 [7] 18/-1/-1->19->16|16->19->18/-1/-1 nccl-test-worker-1:15:58 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:17:61 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:17:61 [3] NCCL INFO Trees [0] 58/-1/-1->59->56|56->59->58/-1/-1 [1] 56/-1/-1->59->58|58->59->56/-1/-1 [2] 56/-1/-1->59->58|58->59->56/-1/-1 [3] 58/-1/-1->59->56|56->59->58/-1/-1 [4] 58/48/0->59->56|56->59->58/48/0 [5] 56/-1/-1->59->58|58->59->56/-1/-1 [6] 56/-1/-1->59->58|58->59->56/-1/-1 [7] 58/-1/-1->59->56|56->59->58/-1/-1 nccl-test-worker-0:16:60 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:16:60 [2] NCCL INFO Trees [0] 1/-1/-1->2->3|3->2->1/-1/-1 [1] 3/33/-1->2->1|1->2->3/33/-1 [2] 3/-1/-1->2->1|1->2->3/-1/-1 [3] 1/-1/-1->2->3|3->2->1/-1/-1 [4] 1/-1/-1->2->3|3->2->1/-1/-1 [5] 3/-1/-1->2->1|1->2->3/-1/-1 [6] 3/-1/-1->2->1|1->2->3/-1/-1 [7] 1/-1/-1->2->3|3->2->1/-1/-1 nccl-test-worker-2:18:55 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:18:55 [4] NCCL INFO Trees [0] -1/-1/-1->20->23|23->20->-1/-1/-1 [1] 23/-1/-1->20->16|16->20->23/-1/-1 [2] 23/-1/-1->20->39|39->20->23/-1/-1 [3] 16/-1/-1->20->23|23->20->16/-1/-1 [4] -1/-1/-1->20->23|23->20->-1/-1/-1 [5] 23/-1/-1->20->16|16->20->23/-1/-1 [6] 23/-1/-1->20->31|31->20->23/-1/-1 [7] 16/-1/-1->20->23|23->20->16/-1/-1 nccl-test-worker-1:16:57 [2] NCCL INFO Trees [0] 9/-1/-1->10->11|11->10->9/-1/-1 [1] 11/-1/-1->10->9|9->10->11/-1/-1 [2] 11/-1/-1->10->9|9->10->11/-1/-1 [3] 9/-1/-1->10->11|11->10->9/-1/-1 [4] 9/-1/-1->10->11|11->10->9/-1/-1 [5] 11/41/-1->10->9|9->10->11/41/-1 [6] 11/-1/-1->10->9|9->10->11/-1/-1 [7] 9/-1/-1->10->11|11->10->9/-1/-1 nccl-test-worker-7:16:58 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:16:58 [2] NCCL INFO Trees [0] 57/-1/-1->58->59|59->58->57/-1/-1 [1] 59/-1/-1->58->57|57->58->59/-1/-1 [2] 59/-1/-1->58->57|57->58->59/-1/-1 [3] 57/-1/-1->58->59|59->58->57/-1/-1 [4] 57/-1/-1->58->59|59->58->57/-1/-1 [5] 59/49/1->58->57|57->58->59/49/1 [6] 59/-1/-1->58->57|57->58->59/-1/-1 [7] 57/-1/-1->58->59|59->58->57/-1/-1 nccl-test-worker-0:21:61 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:21:61 [7] NCCL INFO Trees [0] 4/-1/-1->7->6|6->7->4/-1/-1 [1] 6/-1/-1->7->4|4->7->6/-1/-1 [2] 6/36/-1->7->4|4->7->6/36/-1 [3] 4/-1/-1->7->6|6->7->4/-1/-1 [4] 4/-1/-1->7->6|6->7->4/-1/-1 [5] 6/-1/-1->7->4|4->7->6/-1/-1 [6] 6/-1/-1->7->4|4->7->6/-1/-1 [7] 4/-1/-1->7->6|6->7->4/-1/-1 nccl-test-worker-2:20:59 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:20:59 [6] NCCL INFO Trees [0] 23/-1/-1->22->21|21->22->23/-1/-1 [1] 21/-1/-1->22->23|23->22->21/-1/-1 [2] 21/-1/-1->22->23|23->22->21/-1/-1 [3] 23/13/29->22->21|21->22->23/13/29 [4] 23/-1/-1->22->21|21->22->23/-1/-1 [5] 21/-1/-1->22->23|23->22->21/-1/-1 [6] 21/-1/-1->22->23|23->22->21/-1/-1 [7] 23/-1/-1->22->21|21->22->23/-1/-1 nccl-test-worker-1:14:55 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-1:14:55 [0] NCCL INFO Trees [0] 11/-1/-1->8->19|19->8->11/-1/-1 [1] 12/-1/-1->8->11|11->8->12/-1/-1 [2] -1/-1/-1->8->11|11->8->-1/-1/-1 [3] 11/-1/-1->8->12|12->8->11/-1/-1 [4] 11/-1/-1->8->-1|-1->8->11/-1/-1 [5] 12/-1/-1->8->11|11->8->12/-1/-1 [6] -1/-1/-1->8->11|11->8->-1/-1/-1 [7] 11/-1/-1->8->12|12->8->11/-1/-1 nccl-test-worker-7:14:56 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:14:56 [0] NCCL INFO Trees [0] 59/-1/-1->56->51|51->56->59/-1/-1 [1] 60/-1/-1->56->59|59->56->60/-1/-1 [2] -1/-1/-1->56->59|59->56->-1/-1/-1 [3] 59/-1/-1->56->60|60->56->59/-1/-1 [4] 59/-1/-1->56->43|43->56->59/-1/-1 [5] 60/-1/-1->56->59|59->56->60/-1/-1 [6] -1/-1/-1->56->59|59->56->-1/-1/-1 [7] 59/-1/-1->56->60|60->56->59/-1/-1 nccl-test-worker-0:17:62 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:17:62 [3] NCCL INFO Trees [0] 2/32/-1->3->0|0->3->2/32/-1 [1] 0/-1/-1->3->2|2->3->0/-1/-1 [2] 0/-1/-1->3->2|2->3->0/-1/-1 [3] 2/-1/-1->3->0|0->3->2/-1/-1 [4] 2/-1/-1->3->0|0->3->2/-1/-1 [5] 0/-1/-1->3->2|2->3->0/-1/-1 [6] 0/-1/-1->3->2|2->3->0/-1/-1 [7] 2/-1/-1->3->0|0->3->2/-1/-1 nccl-test-worker-2:19:61 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:19:61 [5] NCCL INFO Trees [0] 22/-1/-1->21->17|17->21->22/-1/-1 [1] -1/-1/-1->21->22|22->21->-1/-1/-1 [2] 17/-1/-1->21->22|22->21->17/-1/-1 [3] 22/-1/-1->21->38|38->21->22/-1/-1 [4] 22/-1/-1->21->17|17->21->22/-1/-1 [5] -1/-1/-1->21->22|22->21->-1/-1/-1 [6] 17/-1/-1->21->22|22->21->17/-1/-1 [7] 22/-1/-1->21->30|30->21->22/-1/-1 nccl-test-worker-1:15:58 [1] NCCL INFO Trees [0] 13/-1/-1->9->10|10->9->13/-1/-1 [1] 10/-1/-1->9->18|18->9->10/-1/-1 [2] 10/-1/-1->9->13|13->9->10/-1/-1 [3] -1/-1/-1->9->10|10->9->-1/-1/-1 [4] 13/-1/-1->9->10|10->9->13/-1/-1 [5] 10/-1/-1->9->-1|-1->9->10/-1/-1 [6] 10/-1/-1->9->13|13->9->10/-1/-1 [7] -1/-1/-1->9->10|10->9->-1/-1/-1 nccl-test-worker-7:25:54 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-7:25:54 [7] NCCL INFO Trees [0] 60/-1/-1->63->62|62->63->60/-1/-1 [1] 62/-1/-1->63->60|60->63->62/-1/-1 [2] 62/-1/-1->63->60|60->63->62/-1/-1 [3] 60/-1/-1->63->62|62->63->60/-1/-1 [4] 60/-1/-1->63->62|62->63->60/-1/-1 [5] 62/-1/-1->63->60|60->63->62/-1/-1 [6] 62/52/4->63->60|60->63->62/52/4 [7] 60/-1/-1->63->62|62->63->60/-1/-1 nccl-test-worker-0:14:55 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-0:14:55 [0] NCCL INFO Trees [0] 3/-1/-1->0->-1|-1->0->3/-1/-1 [1] 4/-1/-1->0->3|3->0->4/-1/-1 [2] -1/-1/-1->0->3|3->0->-1/-1/-1 [3] 3/-1/-1->0->4|4->0->3/-1/-1 [4] 3/-1/-1->0->59|59->0->3/-1/-1 [5] 4/-1/-1->0->3|3->0->4/-1/-1 [6] -1/-1/-1->0->3|3->0->-1/-1/-1 [7] 3/-1/-1->0->4|4->0->3/-1/-1 nccl-test-worker-2:21:60 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-2:21:60 [7] NCCL INFO Trees [0] 20/-1/-1->23->22|22->23->20/-1/-1 [1] 22/-1/-1->23->20|20->23->22/-1/-1 [2] 22/12/28->23->20|20->23->22/12/28 [3] 20/-1/-1->23->22|22->23->20/-1/-1 [4] 20/-1/-1->23->22|22->23->20/-1/-1 [5] 22/-1/-1->23->20|20->23->22/-1/-1 [6] 22/-1/-1->23->20|20->23->22/-1/-1 [7] 20/-1/-1->23->22|22->23->20/-1/-1 nccl-test-worker-3:18:55 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:18:55 [4] NCCL INFO Trees [0] -1/-1/-1->28->31|31->28->-1/-1/-1 [1] 31/-1/-1->28->24|24->28->31/-1/-1 [2] 31/-1/-1->28->23|23->28->31/-1/-1 [3] 24/-1/-1->28->31|31->28->24/-1/-1 [4] -1/-1/-1->28->31|31->28->-1/-1/-1 [5] 31/-1/-1->28->24|24->28->31/-1/-1 [6] 31/-1/-1->28->47|47->28->31/-1/-1 [7] 24/-1/-1->28->31|31->28->24/-1/-1 nccl-test-worker-3:15:57 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:15:57 [1] NCCL INFO Trees [0] 29/-1/-1->25->26|26->25->29/-1/-1 [1] 26/-1/-1->25->18|18->25->26/-1/-1 [2] 26/-1/-1->25->29|29->25->26/-1/-1 [3] -1/-1/-1->25->26|26->25->-1/-1/-1 [4] 29/-1/-1->25->26|26->25->29/-1/-1 [5] 26/-1/-1->25->42|42->25->26/-1/-1 [6] 26/-1/-1->25->29|29->25->26/-1/-1 [7] -1/-1/-1->25->26|26->25->-1/-1/-1 nccl-test-worker-3:17:61 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:17:61 [3] NCCL INFO Trees [0] 26/-1/-1->27->24|24->27->26/-1/-1 [1] 24/-1/-1->27->26|26->27->24/-1/-1 [2] 24/-1/-1->27->26|26->27->24/-1/-1 [3] 26/-1/-1->27->24|24->27->26/-1/-1 [4] 26/16/32->27->24|24->27->26/16/32 [5] 24/-1/-1->27->26|26->27->24/-1/-1 [6] 24/-1/-1->27->26|26->27->24/-1/-1 [7] 26/-1/-1->27->24|24->27->26/-1/-1 nccl-test-worker-3:16:58 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:16:58 [2] NCCL INFO Trees [0] 25/-1/-1->26->27|27->26->25/-1/-1 [1] 27/-1/-1->26->25|25->26->27/-1/-1 [2] 27/-1/-1->26->25|25->26->27/-1/-1 [3] 25/-1/-1->26->27|27->26->25/-1/-1 [4] 25/-1/-1->26->27|27->26->25/-1/-1 [5] 27/17/33->26->25|25->26->27/17/33 [6] 27/-1/-1->26->25|25->26->27/-1/-1 [7] 25/-1/-1->26->27|27->26->25/-1/-1 nccl-test-worker-3:14:59 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:14:59 [0] NCCL INFO Trees [0] 27/-1/-1->24->19|19->24->27/-1/-1 [1] 28/-1/-1->24->27|27->24->28/-1/-1 [2] -1/-1/-1->24->27|27->24->-1/-1/-1 [3] 27/-1/-1->24->28|28->24->27/-1/-1 [4] 27/-1/-1->24->43|43->24->27/-1/-1 [5] 28/-1/-1->24->27|27->24->28/-1/-1 [6] -1/-1/-1->24->27|27->24->-1/-1/-1 [7] 27/-1/-1->24->28|28->24->27/-1/-1 nccl-test-worker-3:19:60 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:19:60 [5] NCCL INFO Trees [0] 30/-1/-1->29->25|25->29->30/-1/-1 [1] -1/-1/-1->29->30|30->29->-1/-1/-1 [2] 25/-1/-1->29->30|30->29->25/-1/-1 [3] 30/-1/-1->29->22|22->29->30/-1/-1 [4] 30/-1/-1->29->25|25->29->30/-1/-1 [5] -1/-1/-1->29->30|30->29->-1/-1/-1 [6] 25/-1/-1->29->30|30->29->25/-1/-1 [7] 30/-1/-1->29->46|46->29->30/-1/-1 nccl-test-worker-3:21:54 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:21:54 [6] NCCL INFO Trees [0] 31/-1/-1->30->29|29->30->31/-1/-1 [1] 29/-1/-1->30->31|31->30->29/-1/-1 [2] 29/-1/-1->30->31|31->30->29/-1/-1 [3] 31/-1/-1->30->29|29->30->31/-1/-1 [4] 31/-1/-1->30->29|29->30->31/-1/-1 [5] 29/-1/-1->30->31|31->30->29/-1/-1 [6] 29/-1/-1->30->31|31->30->29/-1/-1 [7] 31/21/37->30->29|29->30->31/21/37 nccl-test-worker-3:24:56 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-3:24:56 [7] NCCL INFO Trees [0] 28/-1/-1->31->30|30->31->28/-1/-1 [1] 30/-1/-1->31->28|28->31->30/-1/-1 [2] 30/-1/-1->31->28|28->31->30/-1/-1 [3] 28/-1/-1->31->30|30->31->28/-1/-1 [4] 28/-1/-1->31->30|30->31->28/-1/-1 [5] 30/-1/-1->31->28|28->31->30/-1/-1 [6] 30/20/36->31->28|28->31->30/20/36 [7] 28/-1/-1->31->30|30->31->28/-1/-1 nccl-test-worker-4:15:58 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:15:58 [1] NCCL INFO Trees [0] 37/-1/-1->33->34|34->33->37/-1/-1 [1] 34/-1/-1->33->2|2->33->34/-1/-1 [2] 34/-1/-1->33->37|37->33->34/-1/-1 [3] -1/-1/-1->33->34|34->33->-1/-1/-1 [4] 37/-1/-1->33->34|34->33->37/-1/-1 [5] 34/-1/-1->33->26|26->33->34/-1/-1 [6] 34/-1/-1->33->37|37->33->34/-1/-1 [7] -1/-1/-1->33->34|34->33->-1/-1/-1 nccl-test-worker-4:17:57 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:17:57 [2] NCCL INFO Trees [0] 33/-1/-1->34->35|35->34->33/-1/-1 [1] 35/17/49->34->33|33->34->35/17/49 [2] 35/-1/-1->34->33|33->34->35/-1/-1 [3] 33/-1/-1->34->35|35->34->33/-1/-1 [4] 33/-1/-1->34->35|35->34->33/-1/-1 [5] 35/-1/-1->34->33|33->34->35/-1/-1 [6] 35/-1/-1->34->33|33->34->35/-1/-1 [7] 33/-1/-1->34->35|35->34->33/-1/-1 nccl-test-worker-4:14:61 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:14:61 [0] NCCL INFO Trees [0] 35/-1/-1->32->3|3->32->35/-1/-1 [1] 36/-1/-1->32->35|35->32->36/-1/-1 [2] -1/-1/-1->32->35|35->32->-1/-1/-1 [3] 35/-1/-1->32->36|36->32->35/-1/-1 [4] 35/-1/-1->32->27|27->32->35/-1/-1 [5] 36/-1/-1->32->35|35->32->36/-1/-1 [6] -1/-1/-1->32->35|35->32->-1/-1/-1 [7] 35/-1/-1->32->36|36->32->35/-1/-1 nccl-test-worker-4:19:55 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:19:55 [3] NCCL INFO Trees [0] 34/16/48->35->32|32->35->34/16/48 [1] 32/-1/-1->35->34|34->35->32/-1/-1 [2] 32/-1/-1->35->34|34->35->32/-1/-1 [3] 34/-1/-1->35->32|32->35->34/-1/-1 [4] 34/-1/-1->35->32|32->35->34/-1/-1 [5] 32/-1/-1->35->34|34->35->32/-1/-1 [6] 32/-1/-1->35->34|34->35->32/-1/-1 [7] 34/-1/-1->35->32|32->35->34/-1/-1 nccl-test-worker-4:20:56 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:20:56 [4] NCCL INFO Trees [0] -1/-1/-1->36->39|39->36->-1/-1/-1 [1] 39/-1/-1->36->32|32->36->39/-1/-1 [2] 39/-1/-1->36->7|7->36->39/-1/-1 [3] 32/-1/-1->36->39|39->36->32/-1/-1 [4] -1/-1/-1->36->39|39->36->-1/-1/-1 [5] 39/-1/-1->36->32|32->36->39/-1/-1 [6] 39/-1/-1->36->31|31->36->39/-1/-1 [7] 32/-1/-1->36->39|39->36->32/-1/-1 nccl-test-worker-4:22:54 [5] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:22:54 [5] NCCL INFO Trees [0] 38/-1/-1->37->33|33->37->38/-1/-1 [1] -1/-1/-1->37->38|38->37->-1/-1/-1 [2] 33/-1/-1->37->38|38->37->33/-1/-1 [3] 38/-1/-1->37->6|6->37->38/-1/-1 [4] 38/-1/-1->37->33|33->37->38/-1/-1 [5] -1/-1/-1->37->38|38->37->-1/-1/-1 [6] 33/-1/-1->37->38|38->37->33/-1/-1 [7] 38/-1/-1->37->30|30->37->38/-1/-1 nccl-test-worker-4:24:60 [6] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:24:60 [6] NCCL INFO Trees [0] 39/-1/-1->38->37|37->38->39/-1/-1 [1] 37/-1/-1->38->39|39->38->37/-1/-1 [2] 37/-1/-1->38->39|39->38->37/-1/-1 [3] 39/21/53->38->37|37->38->39/21/53 [4] 39/-1/-1->38->37|37->38->39/-1/-1 [5] 37/-1/-1->38->39|39->38->37/-1/-1 [6] 37/-1/-1->38->39|39->38->37/-1/-1 [7] 39/-1/-1->38->37|37->38->39/-1/-1 nccl-test-worker-4:25:59 [7] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-4:25:59 [7] NCCL INFO Trees [0] 36/-1/-1->39->38|38->39->36/-1/-1 [1] 38/-1/-1->39->36|36->39->38/-1/-1 [2] 38/20/52->39->36|36->39->38/20/52 [3] 36/-1/-1->39->38|38->39->36/-1/-1 [4] 36/-1/-1->39->38|38->39->36/-1/-1 [5] 38/-1/-1->39->36|36->39->38/-1/-1 [6] 38/-1/-1->39->36|36->39->38/-1/-1 [7] 36/-1/-1->39->38|38->39->36/-1/-1 nccl-test-worker-5:14:54 [0] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:14:54 [0] NCCL INFO Trees [0] 43/-1/-1->40->51|51->40->43/-1/-1 [1] 44/-1/-1->40->43|43->40->44/-1/-1 [2] -1/-1/-1->40->43|43->40->-1/-1/-1 [3] 43/-1/-1->40->44|44->40->43/-1/-1 [4] 43/-1/-1->40->11|11->40->43/-1/-1 [5] 44/-1/-1->40->43|43->40->44/-1/-1 [6] -1/-1/-1->40->43|43->40->-1/-1/-1 [7] 43/-1/-1->40->44|44->40->43/-1/-1 nccl-test-worker-5:15:57 [1] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:15:57 [1] NCCL INFO Trees [0] 45/-1/-1->41->42|42->41->45/-1/-1 [1] 42/-1/-1->41->50|50->41->42/-1/-1 [2] 42/-1/-1->41->45|45->41->42/-1/-1 [3] -1/-1/-1->41->42|42->41->-1/-1/-1 [4] 45/-1/-1->41->42|42->41->45/-1/-1 [5] 42/-1/-1->41->10|10->41->42/-1/-1 [6] 42/-1/-1->41->45|45->41->42/-1/-1 [7] -1/-1/-1->41->42|42->41->-1/-1/-1 nccl-test-worker-5:17:56 [3] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:17:56 [3] NCCL INFO Trees [0] 42/-1/-1->43->40|40->43->42/-1/-1 [1] 40/-1/-1->43->42|42->43->40/-1/-1 [2] 40/-1/-1->43->42|42->43->40/-1/-1 [3] 42/-1/-1->43->40|40->43->42/-1/-1 [4] 42/24/56->43->40|40->43->42/24/56 [5] 40/-1/-1->43->42|42->43->40/-1/-1 [6] 40/-1/-1->43->42|42->43->40/-1/-1 [7] 42/-1/-1->43->40|40->43->42/-1/-1 nccl-test-worker-5:16:61 [2] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:16:61 [2] NCCL INFO Trees [0] 41/-1/-1->42->43|43->42->41/-1/-1 [1] 43/-1/-1->42->41|41->42->43/-1/-1 [2] 43/-1/-1->42->41|41->42->43/-1/-1 [3] 41/-1/-1->42->43|43->42->41/-1/-1 [4] 41/-1/-1->42->43|43->42->41/-1/-1 [5] 43/25/57->42->41|41->42->43/25/57 [6] 43/-1/-1->42->41|41->42->43/-1/-1 [7] 41/-1/-1->42->43|43->42->41/-1/-1 nccl-test-worker-5:18:59 [4] NCCL INFO threadThresholds 8/8/64 | 512/8/64 | 8/8/64 nccl-test-worker-5:18:59 [4] NCCL INFO Trees [0] -1/-1/-1->44->47|47->44->-1/-1/-1 [1] 47/-1/-1->44->40|40->44->47/-1/-1 [2] 47/-1/-1->44->55|55->44->47/-1/-1 [3] 40/-1/-1->44->47|47->44->40/-1/-1 [4] -1/-1/-1->44->47|47->44->-1/-1/-1 [5] 47/-1/-1->44->40|40->44->47/-1/-1 [6] 47/-1/-1->44->15|15->44->47/-1/-1 [7] 40/-1/-1->44->47|47->44->40/-1/-1 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 00 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 00 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 00 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 00 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 00 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 00 : 49[170] -> 53[1b0] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 00 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 00 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 00 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 00 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 00 : 57[170] -> 61[1b0] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 00 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 00 : 17[170] -> 21[1b0] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 00 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 00 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 00 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 00 : 25[170] -> 29[1b0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 00 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 00 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 00 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 00 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 00 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 00 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 00 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 00 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 00 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 00 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 00 : 9[170] -> 13[1b0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 00 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 00 : 1[170] -> 5[1b0] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 00 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 00 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 00 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 00 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 00 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 00 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 00 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 00 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 00 : 33[170] -> 37[1b0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 00 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 00 : 44[1a0] -> 48[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-5:15:57 [1] NCCL INFO Ring 00 : 41[170] -> 45[1b0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 00 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 00 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 00 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 00 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 00 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 00 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 00 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 00 : 28[1a0] -> 32[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:14:59 [0] NCCL INFO Ring 00 : 20[1a0] -> 24[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 00 : 12[1a0] -> 16[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-0:14:55 [0] NCCL INFO Ring 00 : 60[1a0] -> 0[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-5:14:54 [0] NCCL INFO Ring 00 : 36[1a0] -> 40[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-5:20:58 [6] NCCL INFO Ring 00 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 00 : 52[1a0] -> 56[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-1:14:55 [0] NCCL INFO Ring 00 : 4[1a0] -> 8[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-7:14:56 [0] NCCL INFO Ring 00 : 52[1a0] -> 56[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-0:18:56 [4] NCCL INFO Ring 00 : 4[1a0] -> 8[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:18:55 [4] NCCL INFO Ring 00 : 20[1a0] -> 24[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-1:18:60 [4] NCCL INFO Ring 00 : 12[1a0] -> 16[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-7:20:57 [4] NCCL INFO Ring 00 : 60[1a0] -> 0[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-3:18:55 [4] NCCL INFO Ring 00 : 28[1a0] -> 32[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 00 : 36[1a0] -> 40[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-5:18:59 [4] NCCL INFO Ring 00 : 44[1a0] -> 48[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 00 : 45[1b0] -> 41[170] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 00 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 00 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 00 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 00 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 00 : 53[1b0] -> 49[170] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 00 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 00 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 00 : 13[1b0] -> 9[170] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 00 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 00 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 00 : 5[1b0] -> 1[170] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 00 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 00 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 00 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 00 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 00 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 00 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 00 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 00 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 00 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 00 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 00 : 21[1b0] -> 17[170] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 00 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 00 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 00 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 00 : 29[1b0] -> 25[170] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 00 : 61[1b0] -> 57[170] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 00 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 00 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 00 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 00 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 00 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 00 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 00 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 00 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 00 : 37[1b0] -> 33[170] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 00 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 00 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 01 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 01 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 01 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 01 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 01 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 01 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 01 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 01 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 01 : 45[1b0] -> 49[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 01 : 13[1b0] -> 17[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 01 : 45[1b0] -> 49[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-3:15:57 [1] NCCL INFO Ring 01 : 21[1b0] -> 25[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-0:15:58 [1] NCCL INFO Ring 01 : 61[1b0] -> 1[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-1:15:58 [1] NCCL INFO Ring 01 : 5[1b0] -> 9[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-6:22:55 [5] NCCL INFO Ring 01 : 53[1b0] -> 57[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-1:19:56 [5] NCCL INFO Ring 01 : 13[1b0] -> 17[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 01 : 21[1b0] -> 25[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:15:58 [1] NCCL INFO Ring 01 : 29[1b0] -> 33[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-0:19:57 [5] NCCL INFO Ring 01 : 5[1b0] -> 9[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-7:15:60 [1] NCCL INFO Ring 01 : 53[1b0] -> 57[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:19:60 [5] NCCL INFO Ring 01 : 29[1b0] -> 33[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-7:22:59 [5] NCCL INFO Ring 01 : 61[1b0] -> 1[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:22:54 [5] NCCL INFO Ring 01 : 37[1b0] -> 41[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-5:15:57 [1] NCCL INFO Ring 01 : 37[1b0] -> 41[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-6:14:57 [0] NCCL INFO Ring 00 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 00 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 00 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 00 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 00 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 00 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 01 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 01 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 01 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 01 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 00 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 01 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 00 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 00 : 40[160] -> 51[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 00 : 16[160] -> 35[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 00 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 00 : 32[160] -> 3[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-2:17:57 [3] NCCL INFO Ring 00 : 8[160] -> 19[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-7:17:61 [3] NCCL INFO Ring 00 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 01 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 01 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 00 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 00 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 01 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 01 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 01 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 01 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 01 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 01 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 01 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 01 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 01 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 00 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 00 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 00 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 00 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 00 : 48[160] -> 35[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-5:18:59 [4] NCCL INFO Ring 01 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 00 : 56[160] -> 51[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-1:18:60 [4] NCCL INFO Ring 00 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 01 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 00 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 00 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 00 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 01 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 01 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 01 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 01 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 00 : 24[160] -> 19[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-2:18:55 [4] NCCL INFO Ring 01 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 01 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 01 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 01 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 01 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 01 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 00 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 01 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 01 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 01 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 01 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 00 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 01 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 01 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 00 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 01 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 01 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 01 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 01 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 01 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 01 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 01 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 00 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 01 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 01 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 01 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 00 : 24[160] -> 19[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:14:57 [0] NCCL INFO Ring 00 : 48[160] -> 35[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:24:60 [6] NCCL INFO Ring 01 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 01 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 00 : 32[160] -> 3[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 01 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 01 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 00 : 16[160] -> 35[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-0:14:55 [0] NCCL INFO Ring 01 : 0[160] -> 4[1a0] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 01 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 01 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 01 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 02 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 01 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 01 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 02 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 01 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 01 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 01 : 4[1a0] -> 0[160] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 02 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 01 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 01 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 02 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 01 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 02 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 02 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 02 : 22[1c0] -> 26[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-1:14:55 [0] NCCL INFO Ring 00 : 8[160] -> 19[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-1:16:57 [2] NCCL INFO Ring 01 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 00 : 35[190] -> 48[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:14:59 [0] NCCL INFO Ring 00 : 19[190] -> 24[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-7:16:58 [2] NCCL INFO Ring 01 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 02 : 62[1c0] -> 2[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 02 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 00 : 35[190] -> 16[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 00 : 3[190] -> 32[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-1:25:61 [6] NCCL INFO Ring 02 : 14[1c0] -> 18[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-7:14:56 [0] NCCL INFO Ring 00 : 56[160] -> 51[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-5:17:56 [3] NCCL INFO Ring 01 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 02 : 54[1c0] -> 58[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-0:20:59 [6] NCCL INFO Ring 02 : 6[1c0] -> 10[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-5:16:61 [2] NCCL INFO Ring 01 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 00 : 40[160] -> 51[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-5:20:58 [6] NCCL INFO Ring 02 : 46[1c0] -> 50[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-3:19:60 [5] NCCL INFO Ring 01 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 00 : 19[190] -> 8[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:15:57 [1] NCCL INFO Ring 01 : 25[170] -> 18[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 01 : 17[170] -> 34[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-1:15:58 [1] NCCL INFO Ring 01 : 9[170] -> 18[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:22:54 [5] NCCL INFO Ring 01 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 02 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 02 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 00 : 3[190] -> 32[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-7:14:56 [0] NCCL INFO Ring 00 : 51[190] -> 56[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 01 : 49[170] -> 34[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-7:15:60 [1] NCCL INFO Ring 01 : 57[170] -> 50[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-5:14:54 [0] NCCL INFO Ring 00 : 51[190] -> 40[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:21:54 [6] NCCL INFO Ring 02 : 30[1c0] -> 34[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 01 : 32[160] -> 36[1a0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 01 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 02 : 38[1c0] -> 42[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 01 : 36[1a0] -> 32[160] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 01 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 02 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 01 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 01 : 33[170] -> 2[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-5:15:57 [1] NCCL INFO Ring 01 : 41[170] -> 50[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-0:16:60 [2] NCCL INFO Ring 01 : 33[170] -> 2[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-0:18:56 [4] NCCL INFO Ring 02 : 4[1a0] -> 0[160] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 02 : 0[160] -> 1[170] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 02 : 3[190] -> 7[1d0] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 02 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 02 : 36[1a0] -> 7[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-0:16:60 [2] NCCL INFO Ring 01 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 02 : 1[170] -> 5[1b0] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 02 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 02 : 5[1b0] -> 1[170] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 00 : 35[190] -> 16[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 00 : 35[190] -> 48[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:17:57 [3] NCCL INFO Ring 00 : 19[190] -> 8[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 01 : 16[160] -> 20[1a0] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 01 : 48[160] -> 52[1a0] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 01 : 2[180] -> 33[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 01 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 01 : 52[1a0] -> 48[160] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 01 : 20[1a0] -> 16[160] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 00 : 19[190] -> 24[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:18:58 [3] NCCL INFO Ring 00 : 51[190] -> 40[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 01 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 02 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 01 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 02 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 01 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 01 : 8[160] -> 12[1a0] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 01 : 24[160] -> 28[1a0] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 02 : 36[1a0] -> 32[160] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 01 : 17[170] -> 34[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 02 : 32[160] -> 33[170] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 00 : 51[190] -> 56[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 02 : 35[190] -> 39[1d0] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 01 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 01 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 01 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 01 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 01 : 12[1a0] -> 8[160] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 01 : 28[1a0] -> 24[160] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 01 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 01 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 01 : 40[160] -> 44[1a0] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 01 : 56[160] -> 60[1a0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 01 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 02 : 20[1a0] -> 16[160] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 02 : 36[1a0] -> 7[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 02 : 16[160] -> 17[170] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 01 : 9[170] -> 18[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:24:56 [7] NCCL INFO Ring 02 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 02 : 20[1a0] -> 39[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-2:17:57 [3] NCCL INFO Ring 02 : 19[190] -> 23[1d0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 02 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 01 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 01 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 01 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 01 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 01 : 44[1a0] -> 40[160] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 02 : 27[190] -> 31[1d0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 01 : 60[1a0] -> 56[160] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 02 : 11[190] -> 15[1d0] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 02 : 28[1a0] -> 24[160] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 02 : 12[1a0] -> 8[160] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 01 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 02 : 6[1c0] -> 10[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-3:16:58 [2] NCCL INFO Ring 02 : 22[1c0] -> 26[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-7:14:56 [0] NCCL INFO Ring 01 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 02 : 24[160] -> 25[170] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 02 : 8[160] -> 9[170] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 02 : 52[1a0] -> 48[160] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 01 : 41[170] -> 50[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-6:14:57 [0] NCCL INFO Ring 02 : 48[160] -> 49[170] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 02 : 20[1a0] -> 39[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:18:58 [3] NCCL INFO Ring 02 : 51[190] -> 55[1d0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 02 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 02 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 02 : 12[1a0] -> 23[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-5:17:56 [3] NCCL INFO Ring 02 : 43[190] -> 47[1d0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 02 : 60[1a0] -> 56[160] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 02 : 59[190] -> 63[1d0] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 02 : 44[1a0] -> 40[160] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 02 : 40[160] -> 41[170] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 02 : 56[160] -> 57[170] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 02 : 54[1c0] -> 58[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-5:16:61 [2] NCCL INFO Ring 02 : 38[1c0] -> 42[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-3:18:55 [4] NCCL INFO Ring 02 : 28[1a0] -> 23[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-1:18:60 [4] NCCL INFO Ring 02 : 12[1a0] -> 23[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:20:61 [4] NCCL INFO Ring 02 : 52[1a0] -> 39[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:25:59 [7] NCCL INFO Ring 02 : 44[1a0] -> 55[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-7:20:57 [4] NCCL INFO Ring 02 : 60[1a0] -> 55[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-5:18:59 [4] NCCL INFO Ring 02 : 44[1a0] -> 55[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-4:17:57 [2] NCCL INFO Ring 01 : 49[170] -> 34[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-4:17:57 [2] NCCL INFO Ring 01 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 02 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 02 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 02 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 02 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 02 : 52[1a0] -> 39[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:15:58 [1] NCCL INFO Ring 01 : 2[180] -> 33[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:16:54 [2] NCCL INFO Ring 01 : 25[170] -> 18[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:16:54 [2] NCCL INFO Ring 01 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 01 : 18[180] -> 9[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-5:16:61 [2] NCCL INFO Ring 02 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 02 : 33[170] -> 37[1b0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 02 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 02 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 01 : 57[170] -> 50[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 02 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 02 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 01 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 02 : 62[1c0] -> 2[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:22:54 [5] NCCL INFO Ring 02 : 37[1b0] -> 33[170] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 01 : 50[180] -> 41[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 01 : 34[180] -> 17[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:15:57 [1] NCCL INFO Ring 01 : 18[180] -> 25[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:21:60 [7] NCCL INFO Ring 02 : 28[1a0] -> 23[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 02 : 7[1d0] -> 36[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 02 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 02 : 60[1a0] -> 55[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 01 : 34[180] -> 49[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-7:15:60 [1] NCCL INFO Ring 01 : 50[180] -> 57[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-0:16:60 [2] NCCL INFO Ring 02 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 02 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 02 : 39[1d0] -> 20[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-2:20:59 [6] NCCL INFO Ring 02 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 02 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 02 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 02 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 02 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 02 : 39[1d0] -> 52[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:24:60 [6] NCCL INFO Ring 02 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 02 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 02 : 23[1d0] -> 12[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:24:60 [6] NCCL INFO Ring 02 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 02 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 02 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 02 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 02 : 55[1d0] -> 44[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-6:24:60 [6] NCCL INFO Ring 02 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 02 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 02 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 02 : 23[1d0] -> 28[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-3:18:55 [4] NCCL INFO Ring 02 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 02 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 02 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 02 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 03 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 03 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 02 : 55[1d0] -> 60[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-0:21:61 [7] NCCL INFO Ring 02 : 7[1d0] -> 36[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-7:20:57 [4] NCCL INFO Ring 02 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 01 : 34[180] -> 17[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:17:57 [2] NCCL INFO Ring 01 : 34[180] -> 49[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-0:21:61 [7] NCCL INFO Ring 03 : 7[1d0] -> 11[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-2:16:54 [2] NCCL INFO Ring 01 : 18[180] -> 9[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 02 : 49[170] -> 53[1b0] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 02 : 17[170] -> 21[1b0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 02 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 02 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 02 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 02 : 30[1c0] -> 34[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 02 : 21[1b0] -> 17[170] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 02 : 53[1b0] -> 49[170] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 01 : 18[180] -> 25[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-6:16:54 [2] NCCL INFO Ring 01 : 50[180] -> 41[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-7:24:55 [6] NCCL INFO Ring 02 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 02 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 02 : 9[170] -> 13[1b0] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 02 : 25[170] -> 29[1b0] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 01 : 50[180] -> 57[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-1:14:55 [0] NCCL INFO Ring 02 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 03 : 63[1d0] -> 3[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-3:14:59 [0] NCCL INFO Ring 02 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 02 : 14[1c0] -> 18[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-1:19:56 [5] NCCL INFO Ring 02 : 13[1b0] -> 9[170] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 02 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 02 : 29[1b0] -> 25[170] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 02 : 41[170] -> 45[1b0] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 02 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 02 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 02 : 57[170] -> 61[1b0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 02 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 02 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 02 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 02 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 02 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 02 : 46[1c0] -> 50[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 02 : 45[1b0] -> 41[170] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 02 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 03 : 24[160] -> 25[170] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 02 : 61[1b0] -> 57[170] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 03 : 8[160] -> 9[170] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 03 : 26[180] -> 30[1c0] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 02 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 03 : 10[180] -> 14[1c0] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 03 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 02 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 03 : 23[1d0] -> 27[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-1:15:58 [1] NCCL INFO Ring 03 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 03 : 7[1d0] -> 11[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-0:14:55 [0] NCCL INFO Ring 03 : 0[160] -> 1[170] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 02 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 03 : 40[160] -> 41[170] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 02 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 03 : 2[180] -> 6[1c0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 03 : 63[1d0] -> 3[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-0:19:57 [5] NCCL INFO Ring 03 : 5[1b0] -> 4[1a0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 03 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 03 : 42[180] -> 46[1c0] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 03 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 03 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 03 : 39[1d0] -> 43[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-7:14:56 [0] NCCL INFO Ring 03 : 56[160] -> 57[170] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 02 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 02 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 03 : 61[1b0] -> 60[1a0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 03 : 55[1d0] -> 59[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-7:16:58 [2] NCCL INFO Ring 03 : 58[180] -> 62[1c0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 02 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 03 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 03 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 03 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 03 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 02 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 03 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 03 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 02 : 23[1d0] -> 12[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-2:16:54 [2] NCCL INFO Ring 02 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 03 : 37[1b0] -> 6[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-6:25:59 [7] NCCL INFO Ring 02 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 03 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 02 : 39[1d0] -> 20[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-2:17:57 [3] NCCL INFO Ring 02 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 03 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 02 : 23[1d0] -> 28[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:25:59 [7] NCCL INFO Ring 02 : 55[1d0] -> 44[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:16:54 [2] NCCL INFO Ring 02 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 02 : 39[1d0] -> 52[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:18:58 [3] NCCL INFO Ring 02 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 03 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 03 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 03 : 24[160] -> 28[1a0] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 03 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 02 : 55[1d0] -> 60[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:20:61 [4] NCCL INFO Ring 03 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 03 : 8[160] -> 12[1a0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 03 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 03 : 23[1d0] -> 27[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-5:17:56 [3] NCCL INFO Ring 03 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 03 : 0[160] -> 4[1a0] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 03 : 40[160] -> 44[1a0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 03 : 39[1d0] -> 43[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-7:20:57 [4] NCCL INFO Ring 03 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 03 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 03 : 4[1a0] -> 0[160] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 03 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 03 : 56[160] -> 60[1a0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 03 : 55[1d0] -> 59[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-7:20:57 [4] NCCL INFO Ring 03 : 60[1a0] -> 56[160] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 03 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 03 : 61[1b0] -> 54[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-3:21:54 [6] NCCL INFO Ring 02 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 02 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 02 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 03 : 29[1b0] -> 28[1a0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 03 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 02 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 03 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 02 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 03 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 02 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 02 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 02 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 03 : 31[1d0] -> 35[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:15:58 [1] NCCL INFO Ring 02 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 03 : 28[1a0] -> 24[160] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 03 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 03 : 13[1b0] -> 12[1a0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 03 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 02 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 03 : 29[1b0] -> 22[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 03 : 32[160] -> 33[170] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 02 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 03 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 03 : 37[1b0] -> 36[1a0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 03 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 03 : 34[180] -> 38[1c0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 02 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 02 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 03 : 31[1d0] -> 35[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-0:21:61 [7] NCCL INFO Ring 03 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 03 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 03 : 15[1d0] -> 19[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-2:17:57 [3] NCCL INFO Ring 02 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 03 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 03 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 03 : 45[1b0] -> 44[1a0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 03 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 03 : 12[1a0] -> 8[160] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 02 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 03 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 03 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 03 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 03 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 03 : 13[1b0] -> 22[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-5:18:59 [4] NCCL INFO Ring 03 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 02 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 03 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 03 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 04 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 03 : 16[160] -> 17[170] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 03 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 02 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 04 : 4[1a0] -> 8[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 03 : 21[1b0] -> 20[1a0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 03 : 47[1d0] -> 51[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:25:59 [7] NCCL INFO Ring 03 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 03 : 18[180] -> 22[1c0] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 03 : 37[1b0] -> 6[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-2:17:57 [3] NCCL INFO Ring 03 : 15[1d0] -> 19[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-4:24:60 [6] NCCL INFO Ring 03 : 21[1b0] -> 38[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 02 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 03 : 44[1a0] -> 40[160] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 03 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 04 : 60[1a0] -> 0[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 03 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 03 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 03 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 03 : 45[1b0] -> 54[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-0:17:62 [3] NCCL INFO Ring 03 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 03 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 03 : 48[160] -> 49[170] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 03 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 03 : 53[1b0] -> 52[1a0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 03 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 03 : 50[180] -> 54[1c0] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 04 : 9[170] -> 13[1b0] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 04 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 04 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 03 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 03 : 47[1d0] -> 51[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-3:16:58 [2] NCCL INFO Ring 03 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 03 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 04 : 1[170] -> 5[1b0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 04 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 04 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 03 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 04 : 4[1a0] -> 8[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-5:16:61 [2] NCCL INFO Ring 03 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 03 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 04 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 04 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 04 : 60[1a0] -> 0[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-2:20:59 [6] NCCL INFO Ring 03 : 13[1b0] -> 22[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-0:16:60 [2] NCCL INFO Ring 04 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 03 : 21[1b0] -> 38[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-0:14:55 [0] NCCL INFO Ring 04 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 04 : 25[170] -> 29[1b0] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 04 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 04 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 03 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 04 : 41[170] -> 45[1b0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 04 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 04 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 04 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 04 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 03 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 04 : 20[1a0] -> 24[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:16:58 [2] NCCL INFO Ring 04 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 03 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 04 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 05 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 03 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 03 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 04 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 04 : 36[1a0] -> 40[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 03 : 32[160] -> 36[1a0] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 04 : 40[160] -> 11[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-6:22:55 [5] NCCL INFO Ring 03 : 53[1b0] -> 38[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-5:14:54 [0] NCCL INFO Ring 04 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 03 : 45[1b0] -> 54[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-0:14:55 [0] NCCL INFO Ring 04 : 0[160] -> 59[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 03 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 03 : 36[1a0] -> 32[160] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 04 : 57[170] -> 61[1b0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 04 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 04 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 04 : 16[160] -> 27[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-7:14:56 [0] NCCL INFO Ring 04 : 52[1a0] -> 56[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-7:16:58 [2] NCCL INFO Ring 04 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 04 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 04 : 24[160] -> 43[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-2:17:57 [3] NCCL INFO Ring 03 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 04 : 36[1a0] -> 40[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 03 : 16[160] -> 20[1a0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 03 : 53[1b0] -> 38[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 03 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 03 : 20[1a0] -> 16[160] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 04 : 48[160] -> 59[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-5:14:54 [0] NCCL INFO Ring 04 : 40[160] -> 11[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:18:58 [3] NCCL INFO Ring 03 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 03 : 48[160] -> 52[1a0] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 03 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 03 : 52[1a0] -> 48[160] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 04 : 20[1a0] -> 24[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:20:59 [6] NCCL INFO Ring 03 : 29[1b0] -> 22[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-4:22:54 [5] NCCL INFO Ring 03 : 6[1c0] -> 37[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 04 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 03 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 04 : 52[1a0] -> 56[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-1:14:55 [0] NCCL INFO Ring 05 : 8[160] -> 12[1a0] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 04 : 24[160] -> 43[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:24:60 [6] NCCL INFO Ring 03 : 61[1b0] -> 54[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-0:20:59 [6] NCCL INFO Ring 03 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 04 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 04 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 04 : 56[160] -> 43[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-0:18:56 [4] NCCL INFO Ring 04 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 04 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 04 : 32[160] -> 27[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:24:56 [7] NCCL INFO Ring 03 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 03 : 6[1c0] -> 37[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-3:17:61 [3] NCCL INFO Ring 04 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 04 : 56[160] -> 43[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-0:16:60 [2] NCCL INFO Ring 05 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 04 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 03 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 03 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 04 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 04 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 04 : 0[160] -> 59[190] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:14:59 [0] NCCL INFO Ring 04 : 43[190] -> 24[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 04 : 11[190] -> 40[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-0:21:61 [7] NCCL INFO Ring 04 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 04 : 5[1b0] -> 1[170] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 03 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 04 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 04 : 11[190] -> 40[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-0:20:59 [6] NCCL INFO Ring 04 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 03 : 38[1c0] -> 21[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 03 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 03 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 04 : 28[1a0] -> 32[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-1:25:61 [6] NCCL INFO Ring 03 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 03 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 03 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 05 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 05 : 40[160] -> 44[1a0] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 04 : 43[190] -> 56[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 05 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 05 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 04 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 03 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 05 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 04 : 33[170] -> 37[1b0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 04 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 04 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 03 : 38[1c0] -> 53[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-6:22:55 [5] NCCL INFO Ring 03 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 03 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 03 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 04 : 28[1a0] -> 32[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-4:17:57 [2] NCCL INFO Ring 04 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 05 : 61[1b0] -> 1[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 04 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 04 : 12[1a0] -> 16[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-0:14:55 [0] NCCL INFO Ring 04 : 59[190] -> 0[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-0:19:57 [5] NCCL INFO Ring 05 : 5[1b0] -> 9[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-0:21:61 [7] NCCL INFO Ring 05 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 05 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 04 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 05 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 04 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 03 : 22[1c0] -> 13[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 04 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 03 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 03 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 03 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 05 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 05 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 04 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 04 : 44[1a0] -> 48[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-1:30:54 [7] NCCL INFO Ring 04 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 04 : 32[160] -> 27[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 03 : 54[1c0] -> 45[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-3:19:60 [5] NCCL INFO Ring 03 : 22[1c0] -> 29[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 04 : 17[170] -> 21[1b0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 04 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 03 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 03 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 04 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 03 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 03 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 04 : 12[1a0] -> 16[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-3:21:54 [6] NCCL INFO Ring 04 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 04 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 04 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 04 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 04 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 04 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 04 : 49[170] -> 53[1b0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 04 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 04 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 04 : 27[190] -> 32[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-6:16:54 [2] NCCL INFO Ring 04 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 05 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 05 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 04 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 03 : 54[1c0] -> 61[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-1:14:55 [0] NCCL INFO Ring 05 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 04 : 44[1a0] -> 48[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-7:22:59 [5] NCCL INFO Ring 03 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 05 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 04 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 04 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 04 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 04 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 04 : 16[160] -> 27[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-5:18:59 [4] NCCL INFO Ring 05 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 04 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 04 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 05 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 05 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 04 : 48[160] -> 59[190] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 04 : 27[190] -> 16[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-6:14:57 [0] NCCL INFO Ring 04 : 59[190] -> 48[160] [receive] via NET/AWS Libfabric/0 nccl-test-worker-4:24:60 [6] NCCL INFO Ring 03 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 04 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 04 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 04 : 43[190] -> 24[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-4:22:54 [5] NCCL INFO Ring 04 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 03 : 38[1c0] -> 21[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:15:58 [1] NCCL INFO Ring 04 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 05 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 03 : 38[1c0] -> 53[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-5:17:56 [3] NCCL INFO Ring 04 : 43[190] -> 56[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-2:20:59 [6] NCCL INFO Ring 03 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 04 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 04 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 05 : 24[160] -> 28[1a0] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 05 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 05 : 56[160] -> 60[1a0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 03 : 22[1c0] -> 13[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:24:60 [6] NCCL INFO Ring 04 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 04 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 05 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 03 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 04 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 04 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 04 : 37[1b0] -> 33[170] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 04 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 05 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 04 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 04 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 03 : 22[1c0] -> 29[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-6:22:55 [5] NCCL INFO Ring 04 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 03 : 54[1c0] -> 45[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 04 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 05 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 05 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 04 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 05 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 05 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 04 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 04 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 05 : 29[1b0] -> 33[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-6:24:60 [6] NCCL INFO Ring 03 : 54[1c0] -> 61[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 04 : 21[1b0] -> 17[170] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 05 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 04 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 04 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 05 : 37[1b0] -> 41[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:25:59 [7] NCCL INFO Ring 05 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 04 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 04 : 13[1b0] -> 9[170] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 04 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 04 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 05 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 04 : 29[1b0] -> 25[170] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 04 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 04 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 05 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 04 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 04 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 05 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 05 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 04 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 04 : 53[1b0] -> 49[170] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 04 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 05 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 05 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 04 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 05 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 04 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 05 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 04 : 45[1b0] -> 41[170] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 04 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 05 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 05 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 05 : 13[1b0] -> 17[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-7:22:59 [5] NCCL INFO Ring 04 : 61[1b0] -> 57[170] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 05 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 04 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 05 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 04 : 27[190] -> 16[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:20:61 [4] NCCL INFO Ring 05 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 05 : 5[1b0] -> 9[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:15:57 [1] NCCL INFO Ring 05 : 21[1b0] -> 25[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:18:55 [4] NCCL INFO Ring 05 : 28[1a0] -> 24[160] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 05 : 21[1b0] -> 25[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-1:18:60 [4] NCCL INFO Ring 05 : 12[1a0] -> 8[160] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 05 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 05 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 05 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 05 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 05 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 05 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 05 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 05 : 13[1b0] -> 17[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-6:25:59 [7] NCCL INFO Ring 05 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 05 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 05 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 05 : 29[1b0] -> 33[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-6:24:60 [6] NCCL INFO Ring 05 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 05 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 05 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 05 : 45[1b0] -> 49[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-1:30:54 [7] NCCL INFO Ring 05 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 05 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 05 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 05 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 05 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 05 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 06 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 04 : 59[190] -> 48[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 05 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 05 : 37[1b0] -> 41[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:19:60 [5] NCCL INFO Ring 05 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 06 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 05 : 53[1b0] -> 57[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-1:25:61 [6] NCCL INFO Ring 05 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 04 : 27[190] -> 32[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-3:24:56 [7] NCCL INFO Ring 06 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 05 : 60[1a0] -> 56[160] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 05 : 53[1b0] -> 57[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-6:25:59 [7] NCCL INFO Ring 05 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 05 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 05 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 05 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 05 : 44[1a0] -> 40[160] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 05 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 05 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 05 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 05 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 06 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 06 : 6[1c0] -> 10[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 05 : 45[1b0] -> 49[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 05 : 17[170] -> 26[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-5:25:55 [7] NCCL INFO Ring 05 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 05 : 33[170] -> 26[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-6:16:54 [2] NCCL INFO Ring 05 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 05 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 05 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 05 : 61[1b0] -> 1[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-1:14:55 [0] NCCL INFO Ring 06 : 8[160] -> 9[170] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 06 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 05 : 25[170] -> 42[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-3:17:61 [3] NCCL INFO Ring 05 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 05 : 16[160] -> 20[1a0] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 05 : 32[160] -> 36[1a0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 06 : 12[1a0] -> 8[160] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 06 : 22[1c0] -> 26[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-7:17:61 [3] NCCL INFO Ring 04 : 59[190] -> 0[160] [send] via NET/AWS Libfabric/0 nccl-test-worker-6:22:55 [5] NCCL INFO Ring 06 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 05 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 06 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 06 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 05 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 06 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 06 : 30[1c0] -> 34[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-3:14:59 [0] NCCL INFO Ring 05 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 05 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 05 : 20[1a0] -> 16[160] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 05 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 05 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 06 : 40[160] -> 41[170] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 06 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO Ring 06 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 05 : 41[170] -> 10[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 06 : 11[190] -> 15[1d0] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 05 : 36[1a0] -> 32[160] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 05 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 06 : 38[1c0] -> 42[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 05 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 05 : 49[170] -> 58[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-7:15:60 [1] NCCL INFO Ring 05 : 57[170] -> 42[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-0:15:58 [1] NCCL INFO Ring 05 : 1[170] -> 58[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:14:61 [0] NCCL INFO Ring 05 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 05 : 0[160] -> 4[1a0] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 05 : 48[160] -> 52[1a0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 05 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 06 : 44[1a0] -> 40[160] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 06 : 54[1c0] -> 58[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-1:25:61 [6] NCCL INFO Ring 06 : 14[1c0] -> 18[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-3:18:55 [4] NCCL INFO Ring 06 : 28[1a0] -> 24[160] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 05 : 17[170] -> 26[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:14:59 [0] NCCL INFO Ring 06 : 24[160] -> 25[170] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 06 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 06 : 27[190] -> 31[1d0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 05 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 06 : 62[1c0] -> 2[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-5:25:55 [7] NCCL INFO Ring 06 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 06 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 05 : 41[170] -> 10[180] [send] via NET/AWS Libfabric/1 nccl-test-worker-0:18:56 [4] NCCL INFO Ring 05 : 4[1a0] -> 0[160] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 05 : 52[1a0] -> 48[160] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 05 : 25[170] -> 42[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-0:17:62 [3] NCCL INFO Ring 05 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 06 : 20[1a0] -> 16[160] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 06 : 19[190] -> 23[1d0] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 05 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 06 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 05 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 06 : 16[160] -> 17[170] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 06 : 36[1a0] -> 32[160] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 06 : 43[190] -> 47[1d0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 06 : 35[190] -> 39[1d0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 05 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 06 : 44[1a0] -> 15[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-6:14:57 [0] NCCL INFO Ring 05 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 06 : 32[160] -> 33[170] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 06 : 14[1c0] -> 18[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:17:57 [2] NCCL INFO Ring 06 : 30[1c0] -> 34[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-2:16:54 [2] NCCL INFO Ring 06 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 06 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 06 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 06 : 46[1c0] -> 50[180] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:25:59 [7] NCCL INFO Ring 06 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 06 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 06 : 28[1a0] -> 47[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-0:21:61 [7] NCCL INFO Ring 06 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 06 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 06 : 60[1a0] -> 56[160] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 05 : 49[170] -> 58[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:16:54 [2] NCCL INFO Ring 06 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 06 : 51[190] -> 55[1d0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 06 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 06 : 4[1a0] -> 0[160] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 06 : 44[1a0] -> 15[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 06 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 06 : 3[190] -> 7[1d0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 06 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 06 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 06 : 56[160] -> 57[170] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 06 : 52[1a0] -> 48[160] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 06 : 59[190] -> 63[1d0] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 06 : 0[160] -> 1[170] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 06 : 46[1c0] -> 50[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-2:18:55 [4] NCCL INFO Ring 06 : 20[1a0] -> 31[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:14:57 [0] NCCL INFO Ring 06 : 48[160] -> 49[170] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 06 : 28[1a0] -> 47[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-3:24:56 [7] NCCL INFO Ring 06 : 20[1a0] -> 31[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 06 : 36[1a0] -> 31[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-5:20:58 [6] NCCL INFO Ring 06 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 06 : 62[1c0] -> 2[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-6:16:54 [2] NCCL INFO Ring 06 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 06 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 06 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 06 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 06 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 06 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 06 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 06 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 06 : 60[1a0] -> 47[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-7:24:55 [6] NCCL INFO Ring 06 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 06 : 52[1a0] -> 63[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-0:18:56 [4] NCCL INFO Ring 06 : 4[1a0] -> 63[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-6:20:61 [4] NCCL INFO Ring 06 : 52[1a0] -> 63[1d0] [send] via NET/AWS Libfabric/2 nccl-test-worker-1:16:57 [2] NCCL INFO Ring 05 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 06 : 9[170] -> 13[1b0] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 06 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 06 : 13[1b0] -> 9[170] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 05 : 33[170] -> 26[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 05 : 26[180] -> 17[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:16:58 [2] NCCL INFO Ring 05 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 05 : 57[170] -> 42[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-5:18:59 [4] NCCL INFO Ring 06 : 15[1d0] -> 44[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-5:16:61 [2] NCCL INFO Ring 05 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 06 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 05 : 26[180] -> 33[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-2:18:55 [4] NCCL INFO Ring 06 : 31[1d0] -> 20[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 05 : 58[180] -> 49[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-5:25:55 [7] NCCL INFO Ring 06 : 60[1a0] -> 47[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-2:18:55 [4] NCCL INFO Ring 06 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 05 : 42[180] -> 25[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:18:55 [4] NCCL INFO Ring 06 : 47[1d0] -> 28[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-7:16:58 [2] NCCL INFO Ring 05 : 1[170] -> 58[180] [receive] via NET/AWS Libfabric/1 nccl-test-worker-3:18:55 [4] NCCL INFO Ring 06 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-5:15:57 [1] NCCL INFO Ring 05 : 10[180] -> 41[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-7:16:58 [2] NCCL INFO Ring 05 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 06 : 36[1a0] -> 31[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-1:16:57 [2] NCCL INFO Ring 05 : 10[180] -> 41[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-1:30:54 [7] NCCL INFO Ring 06 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 06 : 63[1d0] -> 52[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-1:18:60 [4] NCCL INFO Ring 07 : 12[1a0] -> 15[1d0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 07 : 14[1c0] -> 13[1b0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 06 : 4[1a0] -> 63[1d0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-6:20:61 [4] NCCL INFO Ring 06 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 06 : 47[1d0] -> 60[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-5:15:57 [1] NCCL INFO Ring 06 : 41[170] -> 45[1b0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 06 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 05 : 42[180] -> 57[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 06 : 31[1d0] -> 36[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-1:30:54 [7] NCCL INFO Ring 06 : 15[1d0] -> 44[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-4:20:56 [4] NCCL INFO Ring 06 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 05 : 58[180] -> 1[170] [receive] via NET/AWS Libfabric/1 nccl-test-worker-5:14:54 [0] NCCL INFO Ring 06 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 06 : 6[1c0] -> 10[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-5:19:60 [5] NCCL INFO Ring 06 : 45[1b0] -> 41[170] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 06 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 06 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 06 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 06 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 06 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 06 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 07 : 15[1d0] -> 19[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 06 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 06 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 07 : 13[1b0] -> 12[1a0] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 07 : 8[160] -> 9[170] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 07 : 10[180] -> 14[1c0] via P2P/IPC nccl-test-worker-1:15:58 [1] NCCL INFO Ring 07 : 9[170] -> 10[180] via P2P/IPC nccl-test-worker-1:17:59 [3] NCCL INFO Ring 07 : 7[1d0] -> 11[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 07 : 11[190] -> 8[160] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO Ring 07 : 13[1b0] -> 14[1c0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 07 : 10[180] -> 11[190] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 07 : 8[160] -> 12[1a0] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO Ring 07 : 8[160] -> 11[190] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 07 : 45[1b0] -> 14[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-1:18:60 [4] NCCL INFO Ring 07 : 12[1a0] -> 8[160] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 06 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 06 : 63[1d0] -> 4[1a0] [receive] via NET/AWS Libfabric/2 nccl-test-worker-0:18:56 [4] NCCL INFO Ring 06 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 06 : 63[1d0] -> 52[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-7:25:54 [7] NCCL INFO Ring 06 : 63[1d0] -> 4[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-0:21:61 [7] NCCL INFO Ring 07 : 7[1d0] -> 11[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-1:17:59 [3] NCCL INFO Ring 07 : 11[190] -> 10[180] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO Ring 07 : 10[180] -> 9[170] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 05 : 26[180] -> 17[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-0:18:56 [4] NCCL INFO Ring 07 : 4[1a0] -> 7[1d0] via P2P/IPC nccl-test-worker-1:14:55 [0] NCCL INFO comm 0x7eff90000dc0 rank 8 nranks 64 cudaDev 0 busId 160 - Init COMPLETE nccl-test-worker-1:15:58 [1] NCCL INFO comm 0x7f5868000dc0 rank 9 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-1:17:59 [3] NCCL INFO comm 0x7f55fc000dc0 rank 11 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-0:21:61 [7] NCCL INFO Ring 07 : 7[1d0] -> 6[1c0] via P2P/IPC nccl-test-worker-1:16:57 [2] NCCL INFO comm 0x7fc7c8000dc0 rank 10 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-5:16:61 [2] NCCL INFO Ring 05 : 42[180] -> 25[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-5:25:55 [7] NCCL INFO Ring 06 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 07 : 63[1d0] -> 3[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-3:16:58 [2] NCCL INFO Ring 05 : 26[180] -> 33[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-5:18:59 [4] NCCL INFO Ring 07 : 44[1a0] -> 47[1d0] via P2P/IPC nccl-test-worker-5:20:58 [6] NCCL INFO Ring 07 : 46[1c0] -> 45[1b0] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 06 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 06 : 33[170] -> 37[1b0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 05 : 58[180] -> 49[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-2:15:56 [1] NCCL INFO Ring 06 : 17[170] -> 21[1b0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 06 : 47[1d0] -> 28[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-5:16:61 [2] NCCL INFO Ring 05 : 42[180] -> 57[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 06 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 06 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 06 : 37[1b0] -> 33[170] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 06 : 31[1d0] -> 20[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-3:15:57 [1] NCCL INFO Ring 06 : 25[170] -> 29[1b0] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 06 : 22[1c0] -> 26[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 06 : 21[1b0] -> 17[170] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 06 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 06 : 57[170] -> 61[1b0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 05 : 58[180] -> 1[170] [send] via NET/AWS Libfabric/1 nccl-test-worker-4:19:55 [3] NCCL INFO Ring 06 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 06 : 47[1d0] -> 60[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-3:16:58 [2] NCCL INFO Ring 06 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 06 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 06 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 06 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 06 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 06 : 61[1b0] -> 57[170] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 06 : 38[1c0] -> 42[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-6:15:56 [1] NCCL INFO Ring 06 : 49[170] -> 53[1b0] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 07 : 32[160] -> 33[170] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 06 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 06 : 1[170] -> 5[1b0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 07 : 62[1c0] -> 61[1b0] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 06 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 07 : 34[180] -> 38[1c0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 07 : 60[1a0] -> 63[1d0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 06 : 29[1b0] -> 25[170] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 06 : 31[1d0] -> 36[1a0] [send] via NET/AWS Libfabric/2 nccl-test-worker-2:14:58 [0] NCCL INFO Ring 07 : 16[160] -> 17[170] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 06 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 06 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-4:15:58 [1] NCCL INFO Ring 07 : 33[170] -> 34[180] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 06 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 07 : 31[1d0] -> 35[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-5:16:61 [2] NCCL INFO Ring 06 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 06 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 06 : 54[1c0] -> 58[180] [receive] via NET/AWS Libfabric/2 nccl-test-worker-3:17:61 [3] NCCL INFO Ring 06 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 07 : 15[1d0] -> 19[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-6:22:55 [5] NCCL INFO Ring 06 : 53[1b0] -> 49[170] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 06 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 06 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 07 : 28[1a0] -> 31[1d0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 06 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 06 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 06 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 06 : 5[1b0] -> 1[170] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 07 : 15[1d0] -> 14[1c0] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 07 : 35[190] -> 32[160] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 06 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 07 : 18[180] -> 22[1c0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 06 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 07 : 19[190] -> 16[160] via P2P/IPC nccl-test-worker-3:21:54 [6] NCCL INFO Ring 07 : 30[1c0] -> 29[1b0] via P2P/IPC nccl-test-worker-1:30:54 [7] NCCL INFO Ring 07 : 15[1d0] -> 12[1a0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 07 : 47[1d0] -> 51[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-7:17:61 [3] NCCL INFO Ring 06 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 06 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 06 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 06 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-1:18:60 [4] NCCL INFO comm 0x7fe960000dc0 rank 12 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-5:15:57 [1] NCCL INFO Ring 06 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 07 : 32[160] -> 36[1a0] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 06 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 06 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-2:15:56 [1] NCCL INFO Ring 07 : 17[170] -> 18[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 06 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 06 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 07 : 48[160] -> 49[170] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 06 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 07 : 6[1c0] -> 5[1b0] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 07 : 0[160] -> 1[170] via P2P/IPC nccl-test-worker-7:15:60 [1] NCCL INFO Ring 06 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 06 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 07 : 36[1a0] -> 39[1d0] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO Ring 07 : 20[1a0] -> 23[1d0] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 07 : 37[1b0] -> 36[1a0] via P2P/IPC nccl-test-worker-6:18:58 [3] NCCL INFO Ring 07 : 47[1d0] -> 51[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 07 : 21[1b0] -> 20[1a0] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 07 : 45[1b0] -> 44[1a0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 07 : 16[160] -> 20[1a0] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 07 : 42[180] -> 46[1c0] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 06 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO Ring 07 : 38[1c0] -> 37[1b0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 06 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 07 : 63[1d0] -> 3[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-5:14:54 [0] NCCL INFO Ring 07 : 40[160] -> 41[170] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 07 : 29[1b0] -> 28[1a0] via P2P/IPC nccl-test-worker-0:19:57 [5] NCCL INFO Ring 07 : 5[1b0] -> 4[1a0] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 07 : 2[180] -> 6[1c0] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 07 : 26[180] -> 30[1c0] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 07 : 50[180] -> 54[1c0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 07 : 22[1c0] -> 21[1b0] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 07 : 24[160] -> 25[170] via P2P/IPC nccl-test-worker-3:24:56 [7] NCCL INFO Ring 07 : 31[1d0] -> 35[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-6:18:58 [3] NCCL INFO Ring 07 : 51[190] -> 48[160] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 07 : 61[1b0] -> 60[1a0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 07 : 58[180] -> 62[1c0] via P2P/IPC nccl-test-worker-4:17:57 [2] NCCL INFO Ring 07 : 34[180] -> 35[190] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 07 : 3[190] -> 0[160] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 07 : 56[160] -> 57[170] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO Ring 07 : 1[170] -> 2[180] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 07 : 39[1d0] -> 43[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-5:15:57 [1] NCCL INFO Ring 07 : 41[170] -> 42[180] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO Ring 07 : 25[170] -> 26[180] via P2P/IPC nccl-test-worker-6:15:56 [1] NCCL INFO Ring 07 : 49[170] -> 50[180] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 07 : 23[1d0] -> 27[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-2:16:54 [2] NCCL INFO Ring 07 : 18[180] -> 19[190] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 07 : 23[1d0] -> 27[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-6:20:61 [4] NCCL INFO Ring 07 : 52[1a0] -> 55[1d0] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 07 : 39[1d0] -> 43[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-7:15:60 [1] NCCL INFO Ring 07 : 57[170] -> 58[180] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO Ring 07 : 63[1d0] -> 62[1c0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 07 : 47[1d0] -> 46[1c0] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 07 : 55[1d0] -> 59[190] [receive] via NET/AWS Libfabric/3 nccl-test-worker-3:24:56 [7] NCCL INFO Ring 07 : 31[1d0] -> 30[1c0] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 07 : 27[190] -> 24[160] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 07 : 43[190] -> 40[160] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 07 : 53[1b0] -> 52[1a0] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 07 : 42[180] -> 43[190] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 07 : 37[1b0] -> 30[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-6:14:57 [0] NCCL INFO Ring 07 : 48[160] -> 52[1a0] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 07 : 26[180] -> 27[190] via P2P/IPC nccl-test-worker-4:20:56 [4] NCCL INFO Ring 07 : 36[1a0] -> 32[160] via P2P/IPC nccl-test-worker-4:14:61 [0] NCCL INFO Ring 07 : 32[160] -> 35[190] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 07 : 0[160] -> 4[1a0] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 07 : 2[180] -> 3[190] via P2P/IPC nccl-test-worker-7:17:61 [3] NCCL INFO Ring 07 : 59[190] -> 56[160] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO Ring 07 : 54[1c0] -> 53[1b0] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 07 : 58[180] -> 59[190] via P2P/IPC nccl-test-worker-4:19:55 [3] NCCL INFO Ring 07 : 35[190] -> 34[180] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 07 : 23[1d0] -> 22[1c0] via P2P/IPC nccl-test-worker-2:19:61 [5] NCCL INFO Ring 07 : 21[1b0] -> 30[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:17:57 [2] NCCL INFO Ring 07 : 34[180] -> 33[170] via P2P/IPC nccl-test-worker-4:25:59 [7] NCCL INFO Ring 07 : 39[1d0] -> 38[1c0] via P2P/IPC nccl-test-worker-3:14:59 [0] NCCL INFO Ring 07 : 24[160] -> 28[1a0] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 07 : 40[160] -> 44[1a0] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO Ring 07 : 16[160] -> 19[190] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO Ring 07 : 45[1b0] -> 14[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-2:18:55 [4] NCCL INFO Ring 07 : 20[1a0] -> 16[160] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO Ring 07 : 18[180] -> 17[170] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO Ring 07 : 19[190] -> 18[180] via P2P/IPC nccl-test-worker-7:14:56 [0] NCCL INFO Ring 07 : 56[160] -> 60[1a0] via P2P/IPC nccl-test-worker-5:25:55 [7] NCCL INFO Ring 07 : 47[1d0] -> 44[1a0] via P2P/IPC nccl-test-worker-6:16:54 [2] NCCL INFO Ring 07 : 50[180] -> 51[190] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 07 : 29[1b0] -> 46[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-7:25:54 [7] NCCL INFO Ring 07 : 63[1d0] -> 60[1a0] via P2P/IPC nccl-test-worker-7:22:59 [5] NCCL INFO Ring 07 : 61[1b0] -> 46[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:15:58 [1] NCCL INFO comm 0x7fc778000dc0 rank 33 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-4:14:61 [0] NCCL INFO comm 0x7f95bc000dc0 rank 32 nranks 64 cudaDev 0 busId 160 - Init COMPLETE nccl-test-worker-0:19:57 [5] NCCL INFO Ring 07 : 5[1b0] -> 62[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-4:19:55 [3] NCCL INFO comm 0x7f1860000dc0 rank 35 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-3:21:54 [6] NCCL INFO Ring 07 : 21[1b0] -> 30[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-3:24:56 [7] NCCL INFO Ring 07 : 31[1d0] -> 28[1a0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 07 : 53[1b0] -> 62[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-4:17:57 [2] NCCL INFO comm 0x7f2bac000dc0 rank 34 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-2:15:56 [1] NCCL INFO comm 0x7f4088000dc0 rank 17 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-5:20:58 [6] NCCL INFO Ring 07 : 29[1b0] -> 46[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-6:25:59 [7] NCCL INFO Ring 07 : 55[1d0] -> 59[190] [send] via NET/AWS Libfabric/3 nccl-test-worker-7:14:56 [0] NCCL INFO Ring 07 : 56[160] -> 59[190] via P2P/IPC nccl-test-worker-0:20:59 [6] NCCL INFO Ring 07 : 6[1c0] -> 7[1d0] via P2P/IPC nccl-test-worker-0:21:61 [7] NCCL INFO Ring 07 : 7[1d0] -> 4[1a0] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO Ring 07 : 60[1a0] -> 56[160] via P2P/IPC nccl-test-worker-2:14:58 [0] NCCL INFO comm 0x7f9560000dc0 rank 16 nranks 64 cudaDev 0 busId 160 - Init COMPLETE nccl-test-worker-4:25:59 [7] NCCL INFO Ring 07 : 39[1d0] -> 36[1a0] via P2P/IPC nccl-test-worker-2:16:54 [2] NCCL INFO comm 0x7f6c68000dc0 rank 18 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-4:24:60 [6] NCCL INFO Ring 07 : 38[1c0] -> 39[1d0] via P2P/IPC nccl-test-worker-2:17:57 [3] NCCL INFO comm 0x7fcc4c000dc0 rank 19 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-3:14:59 [0] NCCL INFO Ring 07 : 24[160] -> 27[190] via P2P/IPC nccl-test-worker-0:18:56 [4] NCCL INFO Ring 07 : 4[1a0] -> 0[160] via P2P/IPC nccl-test-worker-2:21:60 [7] NCCL INFO Ring 07 : 23[1d0] -> 20[1a0] via P2P/IPC nccl-test-worker-3:18:55 [4] NCCL INFO Ring 07 : 28[1a0] -> 24[160] via P2P/IPC nccl-test-worker-0:14:55 [0] NCCL INFO Ring 07 : 0[160] -> 3[190] via P2P/IPC nccl-test-worker-3:17:61 [3] NCCL INFO Ring 07 : 27[190] -> 26[180] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO Ring 07 : 22[1c0] -> 23[1d0] via P2P/IPC nccl-test-worker-0:16:60 [2] NCCL INFO Ring 07 : 2[180] -> 1[170] via P2P/IPC nccl-test-worker-3:16:58 [2] NCCL INFO Ring 07 : 26[180] -> 25[170] via P2P/IPC nccl-test-worker-0:17:62 [3] NCCL INFO Ring 07 : 3[190] -> 2[180] via P2P/IPC nccl-test-worker-5:17:56 [3] NCCL INFO Ring 07 : 43[190] -> 42[180] via P2P/IPC nccl-test-worker-5:16:61 [2] NCCL INFO Ring 07 : 42[180] -> 41[170] via P2P/IPC nccl-test-worker-5:18:59 [4] NCCL INFO Ring 07 : 44[1a0] -> 40[160] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO Ring 07 : 40[160] -> 43[190] via P2P/IPC nccl-test-worker-7:20:57 [4] NCCL INFO comm 0x7fd880000dc0 rank 60 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-6:16:54 [2] NCCL INFO Ring 07 : 50[180] -> 49[170] via P2P/IPC nccl-test-worker-6:22:55 [5] NCCL INFO Ring 07 : 53[1b0] -> 62[1c0] [send] via NET/AWS Libfabric/3 nccl-test-worker-6:18:58 [3] NCCL INFO Ring 07 : 51[190] -> 50[180] via P2P/IPC nccl-test-worker-2:18:55 [4] NCCL INFO comm 0x7f61fc000dc0 rank 20 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-6:25:59 [7] NCCL INFO Ring 07 : 55[1d0] -> 54[1c0] via P2P/IPC nccl-test-worker-6:20:61 [4] NCCL INFO Ring 07 : 52[1a0] -> 48[160] via P2P/IPC nccl-test-worker-6:14:57 [0] NCCL INFO Ring 07 : 48[160] -> 51[190] via P2P/IPC nccl-test-worker-3:15:57 [1] NCCL INFO comm 0x7f5520000dc0 rank 25 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-7:17:61 [3] NCCL INFO Ring 07 : 59[190] -> 58[180] via P2P/IPC nccl-test-worker-7:16:58 [2] NCCL INFO Ring 07 : 58[180] -> 57[170] via P2P/IPC nccl-test-worker-0:15:58 [1] NCCL INFO comm 0x7f5020000dc0 rank 1 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-0:21:61 [7] NCCL INFO comm 0x7f57cc000dc0 rank 7 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-5:19:60 [5] NCCL INFO Ring 07 : 14[1c0] -> 45[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-5:15:57 [1] NCCL INFO comm 0x7f2368000dc0 rank 41 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-4:20:56 [4] NCCL INFO comm 0x7fc210000dc0 rank 36 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-3:14:59 [0] NCCL INFO comm 0x7fbf6c000dc0 rank 24 nranks 64 cudaDev 0 busId 160 - Init COMPLETE nccl-test-worker-3:18:55 [4] NCCL INFO comm 0x7ff33c000dc0 rank 28 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-3:17:61 [3] NCCL INFO comm 0x7f48d0000dc0 rank 27 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-3:16:58 [2] NCCL INFO comm 0x7f151c000dc0 rank 26 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-0:18:56 [4] NCCL INFO comm 0x7fe1c8000dc0 rank 4 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-7:14:56 [0] NCCL INFO comm 0x7f503c000dc0 rank 56 nranks 64 cudaDev 0 busId 160 - Init COMPLETE nccl-test-worker-0:14:55 [0] NCCL INFO comm 0x7fe1a8000dc0 rank 0 nranks 64 cudaDev 0 busId 160 - Init COMPLETE nccl-test-worker-5:17:56 [3] NCCL INFO comm 0x7f4528000dc0 rank 43 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-0:16:60 [2] NCCL INFO comm 0x7f4d80000dc0 rank 2 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-2:21:60 [7] NCCL INFO comm 0x7f4c60000dc0 rank 23 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-0:17:62 [3] NCCL INFO comm 0x7f303c000dc0 rank 3 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-5:16:61 [2] NCCL INFO comm 0x7f1eec000dc0 rank 42 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-6:15:56 [1] NCCL INFO comm 0x7fe89c000dc0 rank 49 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-5:18:59 [4] NCCL INFO comm 0x7f6b58000dc0 rank 44 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-5:19:60 [5] NCCL INFO Ring 07 : 45[1b0] -> 46[1c0] via P2P/IPC nccl-test-worker-5:14:54 [0] NCCL INFO comm 0x7f9c7c000dc0 rank 40 nranks 64 cudaDev 0 busId 160 - Init COMPLETE nccl-test-worker-7:15:60 [1] NCCL INFO comm 0x7f6e48000dc0 rank 57 nranks 64 cudaDev 1 busId 170 - Init COMPLETE nccl-test-worker-4:25:59 [7] NCCL INFO comm 0x7f8fe0000dc0 rank 39 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-6:18:58 [3] NCCL INFO comm 0x7fd37c000dc0 rank 51 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-6:16:54 [2] NCCL INFO comm 0x7f76c8000dc0 rank 50 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-7:17:61 [3] NCCL INFO comm 0x7fa63c000dc0 rank 59 nranks 64 cudaDev 3 busId 190 - Init COMPLETE nccl-test-worker-7:16:58 [2] NCCL INFO comm 0x7fe380000dc0 rank 58 nranks 64 cudaDev 2 busId 180 - Init COMPLETE nccl-test-worker-6:14:57 [0] NCCL INFO comm 0x7f707c000dc0 rank 48 nranks 64 cudaDev 0 busId 160 - Init COMPLETE # # out-of-place in-place # size count type redop time algbw busbw error time algbw busbw error # (B) (elements) (us) (GB/s) (GB/s) (us) (GB/s) (GB/s) nccl-test-worker-0:14:14 [0] NCCL INFO Launch mode Parallel nccl-test-worker-6:24:60 [6] NCCL INFO Ring 07 : 54[1c0] -> 55[1d0] via P2P/IPC nccl-test-worker-6:25:59 [7] NCCL INFO Ring 07 : 55[1d0] -> 52[1a0] via P2P/IPC nccl-test-worker-1:25:61 [6] NCCL INFO Ring 07 : 14[1c0] -> 15[1d0] via P2P/IPC nccl-test-worker-1:19:56 [5] NCCL INFO comm 0x7feb50000dc0 rank 13 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-1:30:54 [7] NCCL INFO comm 0x7f6acc000dc0 rank 15 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-6:20:61 [4] NCCL INFO comm 0x7ff3dc000dc0 rank 52 nranks 64 cudaDev 4 busId 1a0 - Init COMPLETE nccl-test-worker-6:25:59 [7] NCCL INFO comm 0x7f1fe0000dc0 rank 55 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-1:25:61 [6] NCCL INFO Ring 07 : 14[1c0] -> 45[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-1:25:61 [6] NCCL INFO comm 0x7f72d0000dc0 rank 14 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE nccl-test-worker-6:22:55 [5] NCCL INFO Ring 07 : 62[1c0] -> 53[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-7:24:55 [6] NCCL INFO Ring 07 : 5[1b0] -> 62[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-6:22:55 [5] NCCL INFO Ring 07 : 53[1b0] -> 54[1c0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO Ring 07 : 46[1c0] -> 29[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-5:20:58 [6] NCCL INFO Ring 07 : 61[1b0] -> 46[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-3:19:60 [5] NCCL INFO Ring 07 : 29[1b0] -> 30[1c0] via P2P/IPC nccl-test-worker-6:24:60 [6] NCCL INFO comm 0x7fae2c000dc0 rank 54 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE nccl-test-worker-3:21:54 [6] NCCL INFO Ring 07 : 37[1b0] -> 30[1c0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 07 : 30[1c0] -> 21[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-2:19:61 [5] NCCL INFO Ring 07 : 21[1b0] -> 22[1c0] via P2P/IPC nccl-test-worker-2:20:59 [6] NCCL INFO comm 0x7f768c000dc0 rank 22 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE nccl-test-worker-7:22:59 [5] NCCL INFO Ring 07 : 46[1c0] -> 61[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-7:22:59 [5] NCCL INFO Ring 07 : 61[1b0] -> 62[1c0] via P2P/IPC nccl-test-worker-4:22:54 [5] NCCL INFO Ring 07 : 30[1c0] -> 37[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-4:22:54 [5] NCCL INFO Ring 07 : 37[1b0] -> 38[1c0] via P2P/IPC nccl-test-worker-4:24:60 [6] NCCL INFO comm 0x7fced4000dc0 rank 38 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE nccl-test-worker-7:24:55 [6] NCCL INFO Ring 07 : 62[1c0] -> 63[1d0] via P2P/IPC nccl-test-worker-7:25:54 [7] NCCL INFO comm 0x7fc07c000dc0 rank 63 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-0:19:57 [5] NCCL INFO Ring 07 : 62[1c0] -> 5[1b0] [receive] via NET/AWS Libfabric/3 nccl-test-worker-0:19:57 [5] NCCL INFO Ring 07 : 5[1b0] -> 6[1c0] via P2P/IPC nccl-test-worker-7:24:55 [6] NCCL INFO Ring 07 : 62[1c0] -> 53[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-0:20:59 [6] NCCL INFO comm 0x7f7b20000dc0 rank 6 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE nccl-test-worker-7:24:55 [6] NCCL INFO Ring 07 : 62[1c0] -> 5[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-6:22:55 [5] NCCL INFO comm 0x7f1600000dc0 rank 53 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-0:19:57 [5] NCCL INFO comm 0x7ff46c000dc0 rank 5 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-7:24:55 [6] NCCL INFO comm 0x7f5228000dc0 rank 62 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE nccl-test-worker-5:20:58 [6] NCCL INFO Ring 07 : 46[1c0] -> 47[1d0] via P2P/IPC nccl-test-worker-5:19:60 [5] NCCL INFO comm 0x7fc270000dc0 rank 45 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-5:25:55 [7] NCCL INFO comm 0x7f1db0000dc0 rank 47 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-5:20:58 [6] NCCL INFO Ring 07 : 46[1c0] -> 29[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-5:20:58 [6] NCCL INFO Ring 07 : 46[1c0] -> 61[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-3:21:54 [6] NCCL INFO Ring 07 : 30[1c0] -> 31[1d0] via P2P/IPC nccl-test-worker-3:19:60 [5] NCCL INFO comm 0x7ff478000dc0 rank 29 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-3:24:56 [7] NCCL INFO comm 0x7f339c000dc0 rank 31 nranks 64 cudaDev 7 busId 1d0 - Init COMPLETE nccl-test-worker-7:22:59 [5] NCCL INFO comm 0x7f7600000dc0 rank 61 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-5:20:58 [6] NCCL INFO comm 0x7f076c000dc0 rank 46 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE nccl-test-worker-3:21:54 [6] NCCL INFO Ring 07 : 30[1c0] -> 21[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-3:21:54 [6] NCCL INFO Ring 07 : 30[1c0] -> 37[1b0] [send] via NET/AWS Libfabric/3 nccl-test-worker-2:19:61 [5] NCCL INFO comm 0x7fb6f0000dc0 rank 21 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-4:22:54 [5] NCCL INFO comm 0x7fe7e0000dc0 rank 37 nranks 64 cudaDev 5 busId 1b0 - Init COMPLETE nccl-test-worker-3:21:54 [6] NCCL INFO comm 0x7fc364000dc0 rank 30 nranks 64 cudaDev 6 busId 1c0 - Init COMPLETE 8 2 float sum 2208.6 0.00 0.00 0e+00 2217.0 0.00 0.00 2e-07 16 4 float sum 2222.7 0.00 0.00 2e-07 2222.3 0.00 0.00 2e-07 32 8 float sum 2215.7 0.00 0.00 2e-07 2283.6 0.00 0.00 2e-07 64 16 float sum 2199.0 0.00 0.00 2e-07 2194.6 0.00 0.00 2e-07 128 32 float sum 2195.3 0.00 0.00 2e-07 2188.4 0.00 0.00 2e-07 256 64 float sum 2236.0 0.00 0.00 2e-07 2183.9 0.00 0.00 2e-07 512 128 float sum 2196.9 0.00 0.00 2e-07 2179.9 0.00 0.00 2e-07 1024 256 float sum 2187.5 0.00 0.00 1e-06 2193.3 0.00 0.00 1e-06 2048 512 float sum 2186.7 0.00 0.00 1e-06 2186.4 0.00 0.00 1e-06 4096 1024 float sum 2192.4 0.00 0.00 1e-06 2201.5 0.00 0.00 1e-06 8192 2048 float sum 2197.8 0.00 0.01 1e-06 2200.3 0.00 0.01 1e-06 16384 4096 float sum 2200.6 0.01 0.01 2e-06 2206.9 0.01 0.01 2e-06 32768 8192 float sum 2230.1 0.01 0.03 2e-06 2242.3 0.01 0.03 2e-06 65536 16384 float sum 2268.6 0.03 0.06 2e-06 2259.2 0.03 0.06 2e-06 131072 32768 float sum 2344.7 0.06 0.11 2e-06 2370.7 0.06 0.11 2e-06 262144 65536 float sum 2503.4 0.10 0.21 2e-06 2521.4 0.10 0.20 2e-06 524288 131072 float sum 2587.3 0.20 0.40 2e-06 2573.8 0.20 0.40 2e-06 1048576 262144 float sum 2694.1 0.39 0.77 2e-06 2708.1 0.39 0.76 2e-06 2097152 524288 float sum 2939.8 0.71 1.40 2e-06 2913.3 0.72 1.42 2e-06 4194304 1048576 float sum 3617.8 1.16 2.28 2e-06 3562.1 1.18 2.32 2e-06 8388608 2097152 float sum 16005 0.52 1.03 2e-06 16109 0.52 1.03 2e-06 16777216 4194304 float sum 16921 0.99 1.95 2e-06 16944 0.99 1.95 2e-06 33554432 8388608 float sum 12945 2.59 5.10 2e-06 12858 2.61 5.14 2e-06 67108864 16777216 float sum 22160 3.03 5.96 2e-06 22408 2.99 5.90 2e-06 134217728 33554432 float sum 44121 3.04 5.99 2e-06 44146 3.04 5.99 2e-06 268435456 67108864 float sum 90616 2.96 5.83 2e-06 91616 2.93 5.77 2e-06 536870912 134217728 float sum 186028 2.89 5.68 2e-06 185922 2.89 5.68 2e-06 1073741824 268435456 float sum 402026 2.67 5.26 2e-06 401879 2.67 5.26 2e-06 # Out of bounds values : 0 OK # Avg bus bandwidth : 1.5023 #