ERROR of IMPI 2019.6 test program using the MLX provider in libfabric.

IMPI-2019.6-mlx-provider.out

libfabric:109005:core:core:fi_param_get_():280<info> variable perf_cntr=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable hook=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_max_size=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_max_count=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_merge_regions=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_monitor=<not set>
libfabric:109005:core:mr:ofi_default_cache_size():56<info> default cache size=2074779136
libfabric:109005:core:core:fi_param_get_():280<info> variable provider=<not set>
libfabric:109005:core:core:fi_param_get_():289<info> read string var provider_path=/user/brussel/101/vsc10122/.local/easybuild/software/impi/2019.6.166-iccifort-2020.0.166/intel64/libfabric/lib/prov
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: sockets (2.0)
libfabric:109005:core:core:ofi_reg_dl_prov():584<warn> dlopen(/user/brussel/101/vsc10122/.local/easybuild/software/impi/2019.6.166-iccifort-2020.0.166/intel64/libfabric/lib/prov/libpsmx2-fi.so): libpsm2.so.2: cannot open shared object file: No such file or directory
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable tx_size=<not set>
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable rx_size=<not set>
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable msg_tx_size=<not set>
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable msg_rx_size=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable universe_size=<not set>
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable cm_progress_interval=<not set>
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable buffer_size=<not set>
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: ofi_rxm (1.0)
libfabric:109005:efa:core:fi_param_get_():280<info> variable rx_window_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable tx_max_credits=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable tx_min_credits=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable tx_queue_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable enable_sas_ordering=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable recvwin_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable cq_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable max_memcpy_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable mr_cache_enable=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable mr_cache_merge_regions=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable mr_max_cached_count=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable mr_max_cached_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable mtu_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable tx_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable rx_size=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable tx_iov_limit=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable rx_iov_limit=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable rx_copy_unexp=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable rx_copy_ooo=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable max_timeout=<not set>
libfabric:109005:efa:core:fi_param_get_():280<info> variable timeout_interval=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_max_size=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_max_count=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_merge_regions=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_monitor=<not set>
libfabric:109005:core:mr:ofi_default_cache_size():56<info> default cache size=2074779136
libfabric:109005:tcp:core:fi_param_get_():280<info> variable port_high_range=<not set>
libfabric:109005:tcp:core:fi_param_get_():280<info> variable port_low_range=<not set>
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: tcp (1.0)
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_max_size=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_max_count=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_merge_regions=<not set>
libfabric:109005:core:core:fi_param_get_():280<info> variable mr_cache_monitor=<not set>
libfabric:109005:core:mr:ofi_default_cache_size():56<info> default cache size=2074779136
libfabric:109005:verbs:core:fi_param_get_():280<info> variable tx_size=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable rx_size=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable tx_iov_limit=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable rx_iov_limit=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable inline_size=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable min_rnr_timer=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable prefer_xrc=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable xrcd_filename=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable cqread_bunch_size=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable iface=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable dgram_use_name_server=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable dgram_name_server_port=<not set>
libfabric:109005:verbs:core:fi_param_get_():280<info> variable gid_idx=<not set>
libfabric:109005:verbs:fabric:verbs_devs_print():894<info> list of verbs devices found for FI_EP_MSG:
libfabric:109005:verbs:fabric:verbs_devs_print():898<info> #1 mlx4_0 - IPoIB addresses:
libfabric:109005:verbs:fabric:verbs_devs_print():908<info>      10.143.1.3
libfabric:109005:verbs:fabric:verbs_devs_print():908<info>      fe80::202:c903:fd:b111
libfabric:109005:verbs:fabric:fi_ibv_get_device_attrs():653<info> device mlx4_0: first found active port is 1
libfabric:109005:verbs:fabric:fi_ibv_get_device_attrs():653<info> device mlx4_0: first found active port is 1
libfabric:109005:verbs:fabric:fi_ibv_get_device_attrs():653<info> device mlx4_0: first found active port is 1
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: verbs (1.0)
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: mlx (1.5)
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: ofi_hook_perf (1.0)
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: ofi_hook_debug (1.0)
libfabric:109005:core:core:ofi_register_provider():417<info> registering provider: ofi_hook_noop (1.0)
libfabric:109005:mlx:core:fi_param_get_():280<info> variable inject_limit=<not set>
libfabric:109005:mlx:core:mlx_getinfo():188<info> used inject size = 1024 
libfabric:109005:mlx:core:fi_param_get_():280<info> variable config=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_enable=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_port=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable mr_cache=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ep_flush=<not set>
libfabric:109005:mlx:core:mlx_getinfo():233<info> Loaded MLX version 1.5.1
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable use_srx=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable inject_limit=<not set>
libfabric:109005:mlx:core:mlx_getinfo():188<info> used inject size = 1024 
libfabric:109005:mlx:core:fi_param_get_():280<info> variable config=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_enable=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_port=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable mr_cache=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ep_flush=<not set>
libfabric:109005:mlx:core:mlx_getinfo():233<info> Loaded MLX version 1.5.1
libfabric:109005:core:core:ofi_layering_ok():945<info> Need core provider, skipping ofi_rxm
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets
libfabric:109005:mlx:core:fi_param_get_():280<info> variable inject_limit=<not set>
libfabric:109005:mlx:core:mlx_getinfo():188<info> used inject size = 1024 
libfabric:109005:mlx:core:fi_param_get_():280<info> variable config=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_enable=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_port=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable mr_cache=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ep_flush=<not set>
libfabric:109005:mlx:core:mlx_getinfo():233<info> Loaded MLX version 1.5.1
libfabric:109005:ofi_rxm:core:fi_param_get_():280<info> variable use_srx=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable inject_limit=<not set>
libfabric:109005:mlx:core:mlx_getinfo():188<info> used inject size = 1024 
libfabric:109005:mlx:core:fi_param_get_():280<info> variable config=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_enable=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ns_port=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable mr_cache=<not set>
libfabric:109005:mlx:core:fi_param_get_():280<info> variable ep_flush=<not set>
libfabric:109005:mlx:core:mlx_getinfo():233<info> Loaded MLX version 1.5.1
libfabric:109005:core:core:ofi_layering_ok():945<info> Need core provider, skipping ofi_rxm
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets
libfabric:109005:ofi_rxm:core:rxm_alter_info():266<info> mem_tag_format requested: 0x7000007ffff80000 (note: provider doesn't optimize based on mem_tag_format)
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, verbs has been skipped. To use verbs, please, set FI_PROVIDER=verbs
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, tcp has been skipped. To use tcp, please, set FI_PROVIDER=tcp
libfabric:109005:core:core:fi_getinfo_():1064<info> Since mlx can be used, sockets has been skipped. To use sockets, please, set FI_PROVIDER=sockets
libfabric:109005:mlx:core:mlx_fabric_open():173<info> 
libfabric:109005:core:core:fi_fabric_():1298<info> Opened fabric: mlx
libfabric:109005:core:core:fi_param_get_():280<info> variable universe_size=<not set>
libfabric:109005:mlx:core:mlx_cm_getname_mlx_format():73<info> Loaded UCP address: [161]�i��%�S�
Abort(2140047) on node 0 (rank 0 in comm 0): Fatal error in PMPI_Init: Other MPI error, error stack:
MPIR_Init_thread(703)........: 
MPID_Init(958)...............: 
MPIDI_OFI_mpi_init_hook(1334): 
MPIDU_bc_table_create(444)...: 
libfabric:109005:mlx:core:mlx_cleanup():285<info> provider goes cleanup sequence