0001
0002
0003
0004
0005
0006
0007 REMOTE_HOST=
0008 LIST_DEVS=FALSE
0009
0010 DEBUGFS=${DEBUGFS-/sys/kernel/debug}
0011
0012 PERF_RUN_ORDER=32
0013 MAX_MW_SIZE=0
0014 RUN_DMA_TESTS=
0015 DONT_CLEANUP=
0016 MW_SIZE=65536
0017
0018 function show_help()
0019 {
0020 echo "Usage: $0 [OPTIONS] LOCAL_DEV REMOTE_DEV"
0021 echo "Run tests on a pair of NTB endpoints."
0022 echo
0023 echo "If the NTB device loops back to the same host then,"
0024 echo "just specifying the two PCI ids on the command line is"
0025 echo "sufficient. Otherwise, if the NTB link spans two hosts"
0026 echo "use the -r option to specify the hostname for the remote"
0027 echo "device. SSH will then be used to test the remote side."
0028 echo "An SSH key between the root users of the host would then"
0029 echo "be highly recommended."
0030 echo
0031 echo "Options:"
0032 echo " -C don't cleanup ntb modules on exit"
0033 echo " -h show this help message"
0034 echo " -l list available local and remote PCI ids"
0035 echo " -r REMOTE_HOST specify the remote's hostname to connect"
0036 echo " to for the test (using ssh)"
0037 echo " -m MW_SIZE memory window size for ntb_tool"
0038 echo " (default: $MW_SIZE)"
0039 echo " -d run dma tests for ntb_perf"
0040 echo " -p ORDER total data order for ntb_perf"
0041 echo " (default: $PERF_RUN_ORDER)"
0042 echo " -w MAX_MW_SIZE maxmium memory window size for ntb_perf"
0043 echo
0044 }
0045
0046 function parse_args()
0047 {
0048 OPTIND=0
0049 while getopts "b:Cdhlm:r:p:w:" opt; do
0050 case "$opt" in
0051 C) DONT_CLEANUP=1 ;;
0052 d) RUN_DMA_TESTS=1 ;;
0053 h) show_help; exit 0 ;;
0054 l) LIST_DEVS=TRUE ;;
0055 m) MW_SIZE=${OPTARG} ;;
0056 r) REMOTE_HOST=${OPTARG} ;;
0057 p) PERF_RUN_ORDER=${OPTARG} ;;
0058 w) MAX_MW_SIZE=${OPTARG} ;;
0059 \?)
0060 echo "Invalid option: -$OPTARG" >&2
0061 exit 1
0062 ;;
0063 esac
0064 done
0065 }
0066
0067 parse_args "$@"
0068 shift $((OPTIND-1))
0069 LOCAL_DEV=$1
0070 shift
0071 parse_args "$@"
0072 shift $((OPTIND-1))
0073 REMOTE_DEV=$1
0074 shift
0075 parse_args "$@"
0076
0077 set -e
0078
0079 function _modprobe()
0080 {
0081 modprobe "$@" || return 1
0082
0083 if [[ "$REMOTE_HOST" != "" ]]; then
0084 ssh "$REMOTE_HOST" modprobe "$@" || return 1
0085 fi
0086 }
0087
0088 function split_remote()
0089 {
0090 VPATH=$1
0091 REMOTE=
0092
0093 if [[ "$VPATH" == *":/"* ]]; then
0094 REMOTE=${VPATH%%:*}
0095 VPATH=${VPATH
0096 fi
0097 }
0098
0099 function read_file()
0100 {
0101 split_remote $1
0102 if [[ "$REMOTE" != "" ]]; then
0103 ssh "$REMOTE" cat "$VPATH"
0104 else
0105 cat "$VPATH"
0106 fi
0107 }
0108
0109 function write_file()
0110 {
0111 split_remote $2
0112 VALUE=$1
0113
0114 if [[ "$REMOTE" != "" ]]; then
0115 ssh "$REMOTE" "echo \"$VALUE\" > \"$VPATH\""
0116 else
0117 echo "$VALUE" > "$VPATH"
0118 fi
0119 }
0120
0121 function check_file()
0122 {
0123 split_remote $1
0124
0125 if [[ "$REMOTE" != "" ]]; then
0126 ssh "$REMOTE" "[[ -e ${VPATH} ]]"
0127 else
0128 [[ -e ${VPATH} ]]
0129 fi
0130 }
0131
0132 function subdirname()
0133 {
0134 echo $(basename $(dirname $1)) 2> /dev/null
0135 }
0136
0137 function find_pidx()
0138 {
0139 PORT=$1
0140 PPATH=$2
0141
0142 for ((i = 0; i < 64; i++)); do
0143 PEER_DIR="$PPATH/peer$i"
0144
0145 check_file ${PEER_DIR} || break
0146
0147 PEER_PORT=$(read_file "${PEER_DIR}/port")
0148 if [[ ${PORT} -eq $PEER_PORT ]]; then
0149 echo $i
0150 return 0
0151 fi
0152 done
0153
0154 return 1
0155 }
0156
0157 function port_test()
0158 {
0159 LOC=$1
0160 REM=$2
0161
0162 echo "Running port tests on: $(basename $LOC) / $(basename $REM)"
0163
0164 LOCAL_PORT=$(read_file "$LOC/port")
0165 REMOTE_PORT=$(read_file "$REM/port")
0166
0167 LOCAL_PIDX=$(find_pidx ${REMOTE_PORT} "$LOC")
0168 REMOTE_PIDX=$(find_pidx ${LOCAL_PORT} "$REM")
0169
0170 echo "Local port ${LOCAL_PORT} with index ${REMOTE_PIDX} on remote host"
0171 echo "Peer port ${REMOTE_PORT} with index ${LOCAL_PIDX} on local host"
0172
0173 echo " Passed"
0174 }
0175
0176 function link_test()
0177 {
0178 LOC=$1
0179 REM=$2
0180 EXP=0
0181
0182 echo "Running link tests on: $(subdirname $LOC) / $(subdirname $REM)"
0183
0184 if ! write_file "N" "$LOC/../link" 2> /dev/null; then
0185 echo " Unsupported"
0186 return
0187 fi
0188
0189 write_file "N" "$LOC/link_event"
0190
0191 if [[ $(read_file "$REM/link") != "N" ]]; then
0192 echo "Expected link to be down in $REM/link" >&2
0193 exit -1
0194 fi
0195
0196 write_file "Y" "$LOC/../link"
0197
0198 echo " Passed"
0199 }
0200
0201 function doorbell_test()
0202 {
0203 LOC=$1
0204 REM=$2
0205 EXP=0
0206
0207 echo "Running db tests on: $(basename $LOC) / $(basename $REM)"
0208
0209 DB_VALID_MASK=$(read_file "$LOC/db_valid_mask")
0210
0211 write_file "c $DB_VALID_MASK" "$REM/db"
0212
0213 for ((i = 0; i < 64; i++)); do
0214 DB=$(read_file "$REM/db")
0215 if [[ "$DB" -ne "$EXP" ]]; then
0216 echo "Doorbell doesn't match expected value $EXP " \
0217 "in $REM/db" >&2
0218 exit -1
0219 fi
0220
0221 let "MASK = (1 << $i) & $DB_VALID_MASK" || true
0222 let "EXP = $EXP | $MASK" || true
0223
0224 write_file "s $MASK" "$LOC/peer_db"
0225 done
0226
0227 write_file "c $DB_VALID_MASK" "$REM/db_mask"
0228 write_file $DB_VALID_MASK "$REM/db_event"
0229 write_file "s $DB_VALID_MASK" "$REM/db_mask"
0230
0231 write_file "c $DB_VALID_MASK" "$REM/db"
0232
0233 echo " Passed"
0234 }
0235
0236 function get_files_count()
0237 {
0238 NAME=$1
0239 LOC=$2
0240
0241 split_remote $LOC
0242
0243 if [[ "$REMOTE" == "" ]]; then
0244 echo $(ls -1 "$VPATH"/${NAME}* 2>/dev/null | wc -l)
0245 else
0246 echo $(ssh "$REMOTE" "ls -1 \"$VPATH\"/${NAME}* | \
0247 wc -l" 2> /dev/null)
0248 fi
0249 }
0250
0251 function scratchpad_test()
0252 {
0253 LOC=$1
0254 REM=$2
0255
0256 echo "Running spad tests on: $(subdirname $LOC) / $(subdirname $REM)"
0257
0258 CNT=$(get_files_count "spad" "$LOC")
0259
0260 if [[ $CNT -eq 0 ]]; then
0261 echo " Unsupported"
0262 return
0263 fi
0264
0265 for ((i = 0; i < $CNT; i++)); do
0266 VAL=$RANDOM
0267 write_file "$VAL" "$LOC/spad$i"
0268 RVAL=$(read_file "$REM/../spad$i")
0269
0270 if [[ "$VAL" -ne "$RVAL" ]]; then
0271 echo "Scratchpad $i value $RVAL doesn't match $VAL" >&2
0272 exit -1
0273 fi
0274 done
0275
0276 echo " Passed"
0277 }
0278
0279 function message_test()
0280 {
0281 LOC=$1
0282 REM=$2
0283
0284 echo "Running msg tests on: $(subdirname $LOC) / $(subdirname $REM)"
0285
0286 CNT=$(get_files_count "msg" "$LOC")
0287
0288 if [[ $CNT -eq 0 ]]; then
0289 echo " Unsupported"
0290 return
0291 fi
0292
0293 MSG_OUTBITS_MASK=$(read_file "$LOC/../msg_inbits")
0294 MSG_INBITS_MASK=$(read_file "$REM/../msg_inbits")
0295
0296 write_file "c $MSG_OUTBITS_MASK" "$LOC/../msg_sts"
0297 write_file "c $MSG_INBITS_MASK" "$REM/../msg_sts"
0298
0299 for ((i = 0; i < $CNT; i++)); do
0300 VAL=$RANDOM
0301 write_file "$VAL" "$LOC/msg$i"
0302 RVAL=$(read_file "$REM/../msg$i")
0303
0304 if [[ "$VAL" -ne "${RVAL%%<-*}" ]]; then
0305 echo "Message $i value $RVAL doesn't match $VAL" >&2
0306 exit -1
0307 fi
0308 done
0309
0310 echo " Passed"
0311 }
0312
0313 function get_number()
0314 {
0315 KEY=$1
0316
0317 sed -n "s/^\(${KEY}\)[ \t]*\(0x[0-9a-fA-F]*\)\(\[p\]\)\?$/\2/p"
0318 }
0319
0320 function mw_alloc()
0321 {
0322 IDX=$1
0323 LOC=$2
0324 REM=$3
0325
0326 write_file $MW_SIZE "$LOC/mw_trans$IDX"
0327
0328 INB_MW=$(read_file "$LOC/mw_trans$IDX")
0329 MW_ALIGNED_SIZE=$(echo "$INB_MW" | get_number "Window Size")
0330 MW_DMA_ADDR=$(echo "$INB_MW" | get_number "DMA Address")
0331
0332 write_file "$MW_DMA_ADDR:$(($MW_ALIGNED_SIZE))" "$REM/peer_mw_trans$IDX"
0333
0334 if [[ $MW_SIZE -ne $MW_ALIGNED_SIZE ]]; then
0335 echo "MW $IDX size aligned to $MW_ALIGNED_SIZE"
0336 fi
0337 }
0338
0339 function write_mw()
0340 {
0341 split_remote $2
0342
0343 if [[ "$REMOTE" != "" ]]; then
0344 ssh "$REMOTE" \
0345 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
0346 else
0347 dd if=/dev/urandom "of=$VPATH" 2> /dev/null || true
0348 fi
0349 }
0350
0351 function mw_check()
0352 {
0353 IDX=$1
0354 LOC=$2
0355 REM=$3
0356
0357 write_mw "$LOC/mw$IDX"
0358
0359 split_remote "$LOC/mw$IDX"
0360 if [[ "$REMOTE" == "" ]]; then
0361 A=$VPATH
0362 else
0363 A=/tmp/ntb_test.$$.A
0364 ssh "$REMOTE" cat "$VPATH" > "$A"
0365 fi
0366
0367 split_remote "$REM/peer_mw$IDX"
0368 if [[ "$REMOTE" == "" ]]; then
0369 B=$VPATH
0370 else
0371 B=/tmp/ntb_test.$$.B
0372 ssh "$REMOTE" cat "$VPATH" > "$B"
0373 fi
0374
0375 cmp -n $MW_ALIGNED_SIZE "$A" "$B"
0376 if [[ $? != 0 ]]; then
0377 echo "Memory window $MW did not match!" >&2
0378 fi
0379
0380 if [[ "$A" == "/tmp/*" ]]; then
0381 rm "$A"
0382 fi
0383
0384 if [[ "$B" == "/tmp/*" ]]; then
0385 rm "$B"
0386 fi
0387 }
0388
0389 function mw_free()
0390 {
0391 IDX=$1
0392 LOC=$2
0393 REM=$3
0394
0395 write_file "$MW_DMA_ADDR:0" "$REM/peer_mw_trans$IDX"
0396
0397 write_file 0 "$LOC/mw_trans$IDX"
0398 }
0399
0400 function mw_test()
0401 {
0402 LOC=$1
0403 REM=$2
0404
0405 CNT=$(get_files_count "mw_trans" "$LOC")
0406
0407 for ((i = 0; i < $CNT; i++)); do
0408 echo "Running mw$i tests on: $(subdirname $LOC) / " \
0409 "$(subdirname $REM)"
0410
0411 mw_alloc $i $LOC $REM
0412
0413 mw_check $i $LOC $REM
0414
0415 mw_free $i $LOC $REM
0416
0417 echo " Passed"
0418 done
0419
0420 }
0421
0422 function pingpong_test()
0423 {
0424 LOC=$1
0425 REM=$2
0426
0427 echo "Running ping pong tests on: $(basename $LOC) / $(basename $REM)"
0428
0429 LOC_START=$(read_file "$LOC/count")
0430 REM_START=$(read_file "$REM/count")
0431
0432 sleep 7
0433
0434 LOC_END=$(read_file "$LOC/count")
0435 REM_END=$(read_file "$REM/count")
0436
0437 if [[ $LOC_START == $LOC_END ]] || [[ $REM_START == $REM_END ]]; then
0438 echo "Ping pong counter not incrementing!" >&2
0439 exit 1
0440 fi
0441
0442 echo " Passed"
0443 }
0444
0445 function msi_test()
0446 {
0447 LOC=$1
0448 REM=$2
0449
0450 write_file 1 $LOC/ready
0451
0452 echo "Running MSI interrupt tests on: $(subdirname $LOC) / $(subdirname $REM)"
0453
0454 CNT=$(read_file "$LOC/count")
0455 for ((i = 0; i < $CNT; i++)); do
0456 START=$(read_file $REM/../irq${i}_occurrences)
0457 write_file $i $LOC/trigger
0458 END=$(read_file $REM/../irq${i}_occurrences)
0459
0460 if [[ $(($END - $START)) != 1 ]]; then
0461 echo "MSI did not trigger the interrupt on the remote side!" >&2
0462 exit 1
0463 fi
0464 done
0465
0466 echo " Passed"
0467 }
0468
0469 function perf_test()
0470 {
0471 USE_DMA=$1
0472
0473 if [[ $USE_DMA == "1" ]]; then
0474 WITH="with"
0475 else
0476 WITH="without"
0477 fi
0478
0479 _modprobe ntb_perf total_order=$PERF_RUN_ORDER \
0480 max_mw_size=$MAX_MW_SIZE use_dma=$USE_DMA
0481
0482 echo "Running local perf test $WITH DMA"
0483 write_file "$LOCAL_PIDX" "$LOCAL_PERF/run"
0484 echo -n " "
0485 read_file "$LOCAL_PERF/run"
0486 echo " Passed"
0487
0488 echo "Running remote perf test $WITH DMA"
0489 write_file "$REMOTE_PIDX" "$REMOTE_PERF/run"
0490 echo -n " "
0491 read_file "$REMOTE_PERF/run"
0492 echo " Passed"
0493
0494 _modprobe -r ntb_perf
0495 }
0496
0497 function ntb_tool_tests()
0498 {
0499 LOCAL_TOOL="$DEBUGFS/ntb_tool/$LOCAL_DEV"
0500 REMOTE_TOOL="$REMOTE_HOST:$DEBUGFS/ntb_tool/$REMOTE_DEV"
0501
0502 echo "Starting ntb_tool tests..."
0503
0504 _modprobe ntb_tool
0505
0506 port_test "$LOCAL_TOOL" "$REMOTE_TOOL"
0507
0508 LOCAL_PEER_TOOL="$LOCAL_TOOL/peer$LOCAL_PIDX"
0509 REMOTE_PEER_TOOL="$REMOTE_TOOL/peer$REMOTE_PIDX"
0510
0511 link_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
0512 link_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
0513
0514
0515 write_file "Y" "$LOCAL_PEER_TOOL/link_event"
0516 write_file "Y" "$REMOTE_PEER_TOOL/link_event"
0517
0518 doorbell_test "$LOCAL_TOOL" "$REMOTE_TOOL"
0519 doorbell_test "$REMOTE_TOOL" "$LOCAL_TOOL"
0520
0521 scratchpad_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
0522 scratchpad_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
0523
0524 message_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
0525 message_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
0526
0527 mw_test "$LOCAL_PEER_TOOL" "$REMOTE_PEER_TOOL"
0528 mw_test "$REMOTE_PEER_TOOL" "$LOCAL_PEER_TOOL"
0529
0530 _modprobe -r ntb_tool
0531 }
0532
0533 function ntb_pingpong_tests()
0534 {
0535 LOCAL_PP="$DEBUGFS/ntb_pingpong/$LOCAL_DEV"
0536 REMOTE_PP="$REMOTE_HOST:$DEBUGFS/ntb_pingpong/$REMOTE_DEV"
0537
0538 echo "Starting ntb_pingpong tests..."
0539
0540 _modprobe ntb_pingpong
0541
0542 pingpong_test $LOCAL_PP $REMOTE_PP
0543
0544 _modprobe -r ntb_pingpong
0545 }
0546
0547 function ntb_msi_tests()
0548 {
0549 LOCAL_MSI="$DEBUGFS/ntb_msi_test/$LOCAL_DEV"
0550 REMOTE_MSI="$REMOTE_HOST:$DEBUGFS/ntb_msi_test/$REMOTE_DEV"
0551
0552 echo "Starting ntb_msi_test tests..."
0553
0554 if ! _modprobe ntb_msi_test 2> /dev/null; then
0555 echo " Not doing MSI tests seeing the module is not available."
0556 return
0557 fi
0558
0559 port_test $LOCAL_MSI $REMOTE_MSI
0560
0561 LOCAL_PEER="$LOCAL_MSI/peer$LOCAL_PIDX"
0562 REMOTE_PEER="$REMOTE_MSI/peer$REMOTE_PIDX"
0563
0564 msi_test $LOCAL_PEER $REMOTE_PEER
0565 msi_test $REMOTE_PEER $LOCAL_PEER
0566
0567 _modprobe -r ntb_msi_test
0568 }
0569
0570 function ntb_perf_tests()
0571 {
0572 LOCAL_PERF="$DEBUGFS/ntb_perf/$LOCAL_DEV"
0573 REMOTE_PERF="$REMOTE_HOST:$DEBUGFS/ntb_perf/$REMOTE_DEV"
0574
0575 echo "Starting ntb_perf tests..."
0576
0577 perf_test 0
0578
0579 if [[ $RUN_DMA_TESTS ]]; then
0580 perf_test 1
0581 fi
0582 }
0583
0584 function cleanup()
0585 {
0586 set +e
0587 _modprobe -r ntb_tool 2> /dev/null
0588 _modprobe -r ntb_perf 2> /dev/null
0589 _modprobe -r ntb_pingpong 2> /dev/null
0590 _modprobe -r ntb_transport 2> /dev/null
0591 _modprobe -r ntb_msi_test 2> /dev/null
0592 set -e
0593 }
0594
0595 cleanup
0596
0597 if ! [[ $$DONT_CLEANUP ]]; then
0598 trap cleanup EXIT
0599 fi
0600
0601 if [ "$(id -u)" != "0" ]; then
0602 echo "This script must be run as root" 1>&2
0603 exit 1
0604 fi
0605
0606 if [[ "$LIST_DEVS" == TRUE ]]; then
0607 echo "Local Devices:"
0608 ls -1 /sys/bus/ntb/devices
0609 echo
0610
0611 if [[ "$REMOTE_HOST" != "" ]]; then
0612 echo "Remote Devices:"
0613 ssh $REMOTE_HOST ls -1 /sys/bus/ntb/devices
0614 fi
0615
0616 exit 0
0617 fi
0618
0619 if [[ "$LOCAL_DEV" == $"" ]] || [[ "$REMOTE_DEV" == $"" ]]; then
0620 show_help
0621 exit 1
0622 fi
0623
0624 ntb_tool_tests
0625 echo
0626 ntb_pingpong_tests
0627 echo
0628 ntb_msi_tests
0629 echo
0630 ntb_perf_tests
0631 echo
