{"$id":"6189870ff72e62923f40554f"}"failed""16""1636400853""0""SLOW=yes NAME=ncli AUSTER_CLEANUP=true MGSNID=lm0221@tcp:lm0222@tcp NETTYPE=tcp mds1_HOST=lm0221 MDSDEV1=\/dev\/vdb mds_HOST=lm0221 MDSDEV=\/dev\/vdb mds1failover_HOST=lm0222 mdsfailover_HOST=lm0222 mds2_HOST=lm0222 MDSDEV2=\/dev\/vdc mds2failover_HOST=lm0221 MDSCOUNT=2 ost1_HOST=lm0223 OSTDEV1=\/dev\/vdd ost1failover_HOST=lm0224 ost2_HOST=lm0223 OSTDEV2=\/dev\/vde ost2failover_HOST=lm0224 ost3_HOST=lm0224 OSTDEV3=\/dev\/vdb ost3failover_HOST=lm0223 ost4_HOST=lm0224 OSTDEV4=\/dev\/vdc ost4failover_HOST=lm0223 OSTCOUNT=4 CLIENTS=lm0225 RCLIENTS=\"lm0226,lm0227,lm0228\"   PDSH=\"\/usr\/bin\/pdsh -R ssh -S -w \" ONLY=failover_ost MDS_MOUNT_OPTS=\"-o rw,user_xattr\" OST_MOUNT_OPTS=\"-o user_xattr\" MDSSIZE=0 OSTSIZE=0 FLAKEY=\"false\" MDSJOURNALSIZE=\"22\" MDS_FS_MKFS_OPTS=\"-O large_xattr\" ENABLE_QUOTA=\"yes\" POWER_DOWN=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh destroy\" SERVER_FAILOVER_PERIOD=\"300\" SHARED_DIRECTORY=\"\/shared\/fremont\/test-results\/xperior-dectet\/6236\/\/ice-2-dectet-3\/shared-dir\/\/recovery-mds-scale\" FSTYPE=\"ldiskfs\" MAXFREE=\"20000000\" FAILURE_MODE=\"HARD\" REQFAIL_PERCENT=\"100\" HONOR_EXCEPT=\"y\" mdtest_nFiles=\"50000\" POWER_UP=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh start\" mdtest_iteration=\"5\" DURATION=\"49680\"  \/usr\/lib64\/lustre\/tests\/recovery-mds-scale.sh 2>     \/var\/log\/xperior\/test_stderr.364274.log 1>  \/var\/log\/xperior\/test_stdout.364274.log""no""1"[{"$id":"6189870ef72e62923f4054bd"},{"$id":"6189870ef72e62923f4054bf"},{"$id":"6189870ef72e62923f4054c1"},{"$id":"6189870ef72e62923f4054c4"},{"$id":"6189870ef72e62923f4054c7"},{"$id":"6189870ef72e62923f4054c9"},{"$id":"6189870ef72e62923f4054cb"},{"$id":"6189870ef72e62923f4054cd"},{"$id":"6189870ef72e62923f4054cf"},{"$id":"6189870ef72e62923f4054d2"},{"$id":"6189870ef72e62923f4054d5"},{"$id":"6189870ef72e62923f4054d7"},{"$id":"6189870ef72e62923f4054d9"},{"$id":"6189870ef72e62923f4054db"},{"$id":"6189870ef72e62923f4054dd"},{"$id":"6189870ef72e62923f4054df"},{"$id":"6189870ef72e62923f4054e1"},{"$id":"6189870ef72e62923f4054e4"},{"$id":"6189870ef72e62923f4054e7"},{"$id":"6189870ef72e62923f4054ea"},{"$id":"6189870ef72e62923f4054ed"},{"$id":"6189870ef72e62923f4054f0"},{"$id":"6189870ef72e62923f4054f2"},{"$id":"6189870ef72e62923f4054f4"},{"$id":"6189870ef72e62923f4054f6"},{"$id":"6189870ef72e62923f4054f8"},{"$id":"6189870ef72e62923f4054fa"},{"$id":"6189870ef72e62923f4054fc"},{"$id":"6189870ff72e62923f4054fe"},{"$id":"6189870ff72e62923f405501"},{"$id":"6189870ff72e62923f405503"},{"$id":"6189870ff72e62923f405505"},{"$id":"6189870ff72e62923f405507"},{"$id":"6189870ff72e62923f405509"},{"$id":"6189870ff72e62923f40550b"},{"$id":"6189870ff72e62923f40550d"},{"$id":"6189870ff72e62923f40550f"},{"$id":"6189870ff72e62923f405511"},{"$id":"6189870ff72e62923f405513"},{"$id":"6189870ff72e62923f405516"},{"$id":"6189870ff72e62923f405518"},{"$id":"6189870ff72e62923f40551a"},{"$id":"6189870ff72e62923f40551d"},{"$id":"6189870ff72e62923f40551f"},{"$id":"6189870ff72e62923f405521"},{"$id":"6189870ff72e62923f405524"},{"$id":"6189870ff72e62923f405526"},{"$id":"6189870ff72e62923f405528"},{"$id":"6189870ff72e62923f40552a"},{"$id":"6189870ff72e62923f40552c"},{"$id":"6189870ff72e62923f40552f"},{"$id":"6189870ff72e62923f405531"},{"$id":"6189870ff72e62923f405533"},{"$id":"6189870ff72e62923f405535"},{"$id":"6189870ff72e62923f405537"},{"$id":"6189870ff72e62923f405539"},{"$id":"6189870ff72e62923f40553c"},{"$id":"6189870ff72e62923f40553e"},{"$id":"6189870ff72e62923f405540"},{"$id":"6189870ff72e62923f405543"},{"$id":"6189870ff72e62923f405545"},{"$id":"6189870ff72e62923f405547"},{"$id":"6189870ff72e62923f405549"},{"$id":"6189870ff72e62923f40554b"},{"$id":"6189870ff72e62923f40554d"}]"yes"{"server_packages":"http:\/\/steve-0.dev.cray.com\/storage\/lustre_builds\/regular-build\/master\/full\/458\/","sessionstarttime":1636402935,"srv_hash_id":"05e6ccd","configuration":"dectet_lnet","srv_branch":"2.14.55_05e6ccd","branch":"master:master:lnet-crash:scale","buildurl":"http:\/\/morpheus.dev.cray.com:8080\/job\/xperior-dectet\/SUITES=recovery-mds-scale,label=dectet\/6236\/","test_opts":"","jira_ticket":"","cli_branch":"2.14.55_05e6ccd","testrail_id":"","client_packages":"http:\/\/steve-0.dev.cray.com\/storage\/lustre_builds\/regular-build\/master\/client\/458\/","result_logs":"http:\/\/morpheus.dev.cray.com:8080\/job\/xperior-dectet\/SUITES=recovery-mds-scale,label=dectet\/6236\/\/testReport\/recovery-mds-scale\/failover_ost","cli_hash_id":"05e6ccd","recipients":""}"No_status_found""-1""2.14.55_05e6ccd""http:\/\/wiki.lustre.org\/index.php\/Testing_Lustre_Code""221017""recovery-mds-scale""Lustre recovery-mds-scale tests""1636435459""1""0""1""SLOW=yes NAME=ncli AUSTER_CLEANUP=true MGSNID=lm0221@tcp:lm0222@tcp NETTYPE=tcp mds1_HOST=lm0221 MDSDEV1=\/dev\/vdb mds_HOST=lm0221 MDSDEV=\/dev\/vdb mds1failover_HOST=lm0222 mdsfailover_HOST=lm0222 mds2_HOST=lm0222 MDSDEV2=\/dev\/vdc mds2failover_HOST=lm0221 MDSCOUNT=2 ost1_HOST=lm0223 OSTDEV1=\/dev\/vdd ost1failover_HOST=lm0224 ost2_HOST=lm0223 OSTDEV2=\/dev\/vde ost2failover_HOST=lm0224 ost3_HOST=lm0224 OSTDEV3=\/dev\/vdb ost3failover_HOST=lm0223 ost4_HOST=lm0224 OSTDEV4=\/dev\/vdc ost4failover_HOST=lm0223 OSTCOUNT=4 CLIENTS=lm0225 RCLIENTS=\"lm0226,lm0227,lm0228\"   PDSH=\"\/usr\/bin\/pdsh -R ssh -S -w \" ONLY=failover_ost MDS_MOUNT_OPTS=\"-o rw,user_xattr\" OST_MOUNT_OPTS=\"-o user_xattr\" MDSSIZE=0 OSTSIZE=0 POWER_DOWN=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh destroy\" ENABLE_QUOTA=\"yes\" SERVER_FAILOVER_PERIOD=\"300\" SHARED_DIRECTORY=\"\/shared\/fremont\/test-results\/xperior-dectet\/6236\/\/ice-2-dectet-3\/shared-dir\/\/recovery-mds-scale\" MDS_FS_MKFS_OPTS=\"-O large_xattr\" FLAKEY=\"false\" MDSJOURNALSIZE=\"22\" HONOR_EXCEPT=\"y\" REQFAIL_PERCENT=\"100\" POWER_UP=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh start\" mdtest_nFiles=\"50000\" DURATION=\"49680\" mdtest_iteration=\"5\" FSTYPE=\"ldiskfs\" MAXFREE=\"20000000\" FAILURE_MODE=\"HARD\" FORMAT=yes \/usr\/lib64\/lustre\/tests\/llmount.sh""stress failover""not ok 1  #No_status_found""yes""ReformatBefore: llmountcleanup passed\nReformatBefore: llmount passed\nError while getting 'lctl dk' from [lm0224], exitcode = 1\nCannot copy log file [\/tmp\/xpdiagnostic.1636400900.25624.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400900.75478.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400901.2281.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400901.67935.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400902.13289.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400902.57797.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400903.03272.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400903.55693.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400904.22845.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1636400905.29099.xml]: 256\n""ice-2.dev.cray.com""data\/schemas\/testds.yaml""1636385159"{"FSTYPE":"ldiskfs","HONOR_EXCEPT":"y","mdtest_iteration":"5","MAXFREE":"20000000","SERVER_FAILOVER_PERIOD":"300","REQFAIL_PERCENT":"100","mdtest_nFiles":"50000","MDS_FS_MKFS_OPTS":"-O large_xattr","POWER_UP":"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh start","DURATION":"49680","FAILURE_MODE":"HARD","POWER_DOWN":"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh destroy","FLAKEY":"false","ENABLE_QUOTA":"yes","MDSJOURNALSIZE":"22","SHARED_DIRECTORY":"\/shared\/fremont\/test-results\/xperior-dectet\/6236\/\/ice-2-dectet-3\/shared-dir\/\/recovery-mds-scale"}"4""2.14.55_05e6ccd""recovery-mds-scale.sh""failover_ost""50300"["\/tmp\/test_logs\/*\/recovery-*.run_*.log"]"Xperior::Executor::LustreTests"null"StoreStat StoreConsole GetDiagnostics CustomLogCollector ReformatBefore StacktraceGenerator""failover_ost""50"1636402935"SLOW=yes NAME=ncli AUSTER_CLEANUP=true MGSNID=lm0221@tcp:lm0222@tcp NETTYPE=tcp mds1_HOST=lm0221 MDSDEV1=\/dev\/vdb mds_HOST=lm0221 MDSDEV=\/dev\/vdb mds1failover_HOST=lm0222 mdsfailover_HOST=lm0222 mds2_HOST=lm0222 MDSDEV2=\/dev\/vdc mds2failover_HOST=lm0221 MDSCOUNT=2 ost1_HOST=lm0223 OSTDEV1=\/dev\/vdd ost1failover_HOST=lm0224 ost2_HOST=lm0223 OSTDEV2=\/dev\/vde ost2failover_HOST=lm0224 ost3_HOST=lm0224 OSTDEV3=\/dev\/vdb ost3failover_HOST=lm0223 ost4_HOST=lm0224 OSTDEV4=\/dev\/vdc ost4failover_HOST=lm0223 OSTCOUNT=4 CLIENTS=lm0225 RCLIENTS=\"lm0226,lm0227,lm0228\"   PDSH=\"\/usr\/bin\/pdsh -R ssh -S -w \" ONLY=failover_ost MDS_MOUNT_OPTS=\"-o rw,user_xattr\" OST_MOUNT_OPTS=\"-o user_xattr\" MDSSIZE=0 OSTSIZE=0 MDS_FS_MKFS_OPTS=\"-O large_xattr\" FLAKEY=\"false\" MDSJOURNALSIZE=\"22\" SHARED_DIRECTORY=\"\/shared\/fremont\/test-results\/xperior-dectet\/6236\/\/ice-2-dectet-3\/shared-dir\/\/recovery-mds-scale\" SERVER_FAILOVER_PERIOD=\"300\" POWER_DOWN=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh destroy\" ENABLE_QUOTA=\"yes\" MAXFREE=\"20000000\" FAILURE_MODE=\"HARD\" FSTYPE=\"ldiskfs\" DURATION=\"49680\" mdtest_iteration=\"5\" mdtest_nFiles=\"50000\" POWER_UP=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' ice-2.dev.cray.com virsh start\" REQFAIL_PERCENT=\"100\" HONOR_EXCEPT=\"y\"  \/usr\/lib64\/lustre\/tests\/llmountcleanup.sh""client1""no""failover_ost._recovery-mds-scale.test_failover_ost.run_dd_debug.*.log.*.log""*dd: error writing *dd-file': Input\/output error*""LUS-2395""10718""2""3"0"""""recovery-mds-scale_failover_ost test failed: dd: error writing '\/mnt\/lustre\/d0.dd-lm0606\/dd-file': Input\/output error"{"components_0":"Lustre (Other)"}{"versions_0":"csl_3.0.0-006"}" failover OST ""1"