{"$id":"5ac0872ef72e6250544ca274"}"yes""0""StoreStat StoreSyslog StoreConsole GetDiagnostics CustomLogCollector ReformatBefore""2.10.58_135_g9ba637b"[{"$id":"5ac0872ef72e6250544ca21b"},{"$id":"5ac0872ef72e6250544ca21d"},{"$id":"5ac0872ef72e6250544ca21f"},{"$id":"5ac0872ef72e6250544ca222"},{"$id":"5ac0872ef72e6250544ca224"},{"$id":"5ac0872ef72e6250544ca227"},{"$id":"5ac0872ef72e6250544ca22a"},{"$id":"5ac0872ef72e6250544ca22c"},{"$id":"5ac0872ef72e6250544ca231"},{"$id":"5ac0872ef72e6250544ca233"},{"$id":"5ac0872ef72e6250544ca236"},{"$id":"5ac0872ef72e6250544ca238"},{"$id":"5ac0872ef72e6250544ca23a"},{"$id":"5ac0872ef72e6250544ca23c"},{"$id":"5ac0872ef72e6250544ca23e"},{"$id":"5ac0872ef72e6250544ca240"},{"$id":"5ac0872ef72e6250544ca242"},{"$id":"5ac0872ef72e6250544ca245"},{"$id":"5ac0872ef72e6250544ca247"},{"$id":"5ac0872ef72e6250544ca249"},{"$id":"5ac0872ef72e6250544ca24b"},{"$id":"5ac0872ef72e6250544ca24e"},{"$id":"5ac0872ef72e6250544ca250"},{"$id":"5ac0872ef72e6250544ca252"},{"$id":"5ac0872ef72e6250544ca254"},{"$id":"5ac0872ef72e6250544ca256"},{"$id":"5ac0872ef72e6250544ca259"},{"$id":"5ac0872ef72e6250544ca25b"},{"$id":"5ac0872ef72e6250544ca25d"},{"$id":"5ac0872ef72e6250544ca260"},{"$id":"5ac0872ef72e6250544ca262"},{"$id":"5ac0872ef72e6250544ca264"},{"$id":"5ac0872ef72e6250544ca266"},{"$id":"5ac0872ef72e6250544ca268"},{"$id":"5ac0872ef72e6250544ca26a"},{"$id":"5ac0872ef72e6250544ca26d"},{"$id":"5ac0872ef72e6250544ca26f"},{"$id":"5ac0872ef72e6250544ca271"}]"http:\/\/wiki.lustre.org\/index.php\/Testing_Lustre_Code""Lustre recovery-mds-scale tests""1522585487""no""failed"{"SERVER_FAILOVER_PERIOD":"300","DURATION":"49680","ENABLE_QUOTA":"yes","FAILURE_MODE":"HARD","SHARED_DIRECTORY":"\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/shared-dir\/\/recovery-mds-scale","REQFAIL_PERCENT":"100","MDS_FS_MKFS_OPTS":"-O large_xattr","mdtest_iteration":"5","MDSJOURNALSIZE":"22","MAXFREE":"20000000","POWER_DOWN":"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh destroy","mdtest_nFiles":"50000","POWER_UP":"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh start"}"0"["\/tmp\/test_logs\/*\/recovery-*.run_*.log"]"4""recovery-mds-scale""No_status_found""client1""50""2.7.22.1""SLOW=yes NAME=ncli MGSNID=fre901@tcp:fre902@tcp NETTYPE=tcp mds1_HOST=fre901 MDSDEV1=\/dev\/vdc mds_HOST=fre901 MDSDEV=\/dev\/vdc mds1failover_HOST=fre902 mdsfailover_HOST=fre902 mds2_HOST=fre902 MDSDEV2=\/dev\/vdb mds2failover_HOST=fre901 MDSCOUNT=2 ost1_HOST=fre903 OSTDEV1=\/dev\/vdd ost1failover_HOST=fre904 ost2_HOST=fre903 OSTDEV2=\/dev\/vde ost2failover_HOST=fre904 ost3_HOST=fre904 OSTDEV3=\/dev\/vdb ost3failover_HOST=fre903 ost4_HOST=fre904 OSTDEV4=\/dev\/vdc ost4failover_HOST=fre903 OSTCOUNT=4 CLIENTS=fre905 RCLIENTS=\"fre906,fre907,fre908\"   PDSH=\"\/usr\/bin\/pdsh -R ssh -S -w \" ONLY=failover_mds MDS_MOUNT_OPTS=\"-o rw,user_xattr\" OST_MOUNT_OPTS=\"-o user_xattr\" MDSSIZE=0 OSTSIZE=0 MDS_FS_MKFS_OPTS=\"-O large_xattr\" MDSJOURNALSIZE=\"22\" SERVER_FAILOVER_PERIOD=\"300\" DURATION=\"49680\" mdtest_nFiles=\"50000\" mdtest_iteration=\"5\" MAXFREE=\"20000000\" FAILURE_MODE=\"HARD\" REQFAIL_PERCENT=\"100\" SHARED_DIRECTORY=\"\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/shared-dir\/\/recovery-mds-scale\" ENABLE_QUOTA=\"yes\" POWER_UP=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh start\" POWER_DOWN=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh destroy\"  \/usr\/lib64\/lustre\/tests\/recovery-mds-scale.sh 2>     \/var\/log\/xperior\/test_stderr.62489.log 1>  \/var\/log\/xperior\/test_stdout.62489.log""ReformatBefore: llmountcleanup passed\nReformatBefore: llmount passed\nCannot copy log file [\/tmp\/xpdiagnostic.1522562896.59071.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1522562897.51933.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1522562898.21624.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1522562898.9039.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1522562899.65061.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1522562900.34273.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1522562901.10148.xml]: 256\nCannot copy log file [\/tmp\/xpdiagnostic.1522562901.79104.xml]: 256\n""stress failover""not ok 1  #No_status_found""yes"null"1522562880""SLOW=yes NAME=ncli MGSNID=fre901@tcp:fre902@tcp NETTYPE=tcp mds1_HOST=fre901 MDSDEV1=\/dev\/vdc mds_HOST=fre901 MDSDEV=\/dev\/vdc mds1failover_HOST=fre902 mdsfailover_HOST=fre902 mds2_HOST=fre902 MDSDEV2=\/dev\/vdb mds2failover_HOST=fre901 MDSCOUNT=2 ost1_HOST=fre903 OSTDEV1=\/dev\/vdd ost1failover_HOST=fre904 ost2_HOST=fre903 OSTDEV2=\/dev\/vde ost2failover_HOST=fre904 ost3_HOST=fre904 OSTDEV3=\/dev\/vdb ost3failover_HOST=fre903 ost4_HOST=fre904 OSTDEV4=\/dev\/vdc ost4failover_HOST=fre903 OSTCOUNT=4 CLIENTS=fre905 RCLIENTS=\"fre906,fre907,fre908\"   PDSH=\"\/usr\/bin\/pdsh -R ssh -S -w \" ONLY=failover_mds MDS_MOUNT_OPTS=\"-o rw,user_xattr\" OST_MOUNT_OPTS=\"-o user_xattr\" MDSSIZE=0 OSTSIZE=0 MDSJOURNALSIZE=\"22\" MDS_FS_MKFS_OPTS=\"-O large_xattr\" DURATION=\"49680\" SERVER_FAILOVER_PERIOD=\"300\" mdtest_iteration=\"5\" MAXFREE=\"20000000\" mdtest_nFiles=\"50000\" FAILURE_MODE=\"HARD\" REQFAIL_PERCENT=\"100\" POWER_DOWN=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh destroy\" POWER_UP=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh start\" ENABLE_QUOTA=\"yes\" SHARED_DIRECTORY=\"\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/shared-dir\/\/recovery-mds-scale\" FORMAT=yes \/usr\/lib64\/lustre\/tests\/llmount.sh""-1""1""1522535187""1"{"srv_hash_id":"6abb8d3","sessionstarttime":1522566955,"result_logs":"http:\/\/morpheus.dev.cray.com:8080\/job\/xperior-octet\/SUITES=recovery-mds-scale,label=octet\/157\/\/testReport\/recovery-mds-scale\/failover_mds","branch":"bns3x:master:octet-L300:scale","cli_branch":"2.10.58_135_g9ba637b","configuration":"octet_L300","cli_hash_id":"1.el7.centos","server_packages":"http:\/\/steve-0.dev.cray.com\/storage\/lustre_builds\/Regular\/b_neo_stable_3.x\/full\/el7\/latest","recipients":"","buildurl":"http:\/\/morpheus.dev.cray.com:8080\/job\/xperior-octet\/SUITES=recovery-mds-scale,label=octet\/157\/","srv_branch":"2.7.22.1","test_opts":"","client_packages":"http:\/\/steve-0.dev.cray.com\/storage\/lustre_builds\/Regular\/master\/client\/el7\/latest\/","jira_ticket":""}"Xperior::Executor::LustreTests""SLOW=yes NAME=ncli MGSNID=fre901@tcp:fre902@tcp NETTYPE=tcp mds1_HOST=fre901 MDSDEV1=\/dev\/vdc mds_HOST=fre901 MDSDEV=\/dev\/vdc mds1failover_HOST=fre902 mdsfailover_HOST=fre902 mds2_HOST=fre902 MDSDEV2=\/dev\/vdb mds2failover_HOST=fre901 MDSCOUNT=2 ost1_HOST=fre903 OSTDEV1=\/dev\/vdd ost1failover_HOST=fre904 ost2_HOST=fre903 OSTDEV2=\/dev\/vde ost2failover_HOST=fre904 ost3_HOST=fre904 OSTDEV3=\/dev\/vdb ost3failover_HOST=fre903 ost4_HOST=fre904 OSTDEV4=\/dev\/vdc ost4failover_HOST=fre903 OSTCOUNT=4 CLIENTS=fre905 RCLIENTS=\"fre906,fre907,fre908\"   PDSH=\"\/usr\/bin\/pdsh -R ssh -S -w \" ONLY=failover_mds MDS_MOUNT_OPTS=\"-o rw,user_xattr\" OST_MOUNT_OPTS=\"-o user_xattr\" MDSSIZE=0 OSTSIZE=0 FAILURE_MODE=\"HARD\" POWER_DOWN=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh destroy\" SHARED_DIRECTORY=\"\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/shared-dir\/\/recovery-mds-scale\" ENABLE_QUOTA=\"yes\" POWER_UP=\"eval ssh -o 'UserKnownHostsFile=\/tmp\/lustre-known-hosts' -o 'StrictHostKeyChecking=no' extropy-kvm9.dev.cray.com virsh start\" REQFAIL_PERCENT=\"100\" MDSJOURNALSIZE=\"22\" MDS_FS_MKFS_OPTS=\"-O large_xattr\" MAXFREE=\"20000000\" mdtest_iteration=\"5\" mdtest_nFiles=\"50000\" DURATION=\"49680\" SERVER_FAILOVER_PERIOD=\"300\"  \/usr\/lib64\/lustre\/tests\/llmountcleanup.sh"{"_recovery-random-scale_pnt_test_fail_client_mds_pnt_run_dd_stdout_pnt_fre906_pnt_log_pnt_fre906":"failover_mds._recovery-random-scale.test_fail_client_mds.run_dd_stdout.fre906.log.fre906.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_debug_log_pnt_fre905_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.debug_log.fre905.1522562874.log","_recovery-mds-scale_pnt_test_failover_mds_pnt_run_dd_stdout_pnt_fre906_pnt_log_pnt_fre906":"failover_mds._recovery-mds-scale.test_failover_mds.run_dd_stdout.fre906.log.fre906.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre904_pnt_log":"failover_mds.messages.fre904.log","_recovery-mds-scale_pnt_test_failover_mds_pnt_run_dbench_stdout_pnt_fre908_pnt_log_pnt_fre908":"failover_mds._recovery-mds-scale.test_failover_mds.run_dbench_stdout.fre908.log.fre908.log","console_pnt_fre908":"failover_mds.console.fre908.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre901_pnt_log":"failover_mds.messages.fre901.log","_recovery-random-scale_pnt_test_fail_client_mds_pnt_run_dbench_stdout_pnt_fre908_pnt_log_pnt_fre908":"failover_mds._recovery-random-scale.test_fail_client_mds.run_dbench_stdout.fre908.log.fre908.log","console_pnt_fre906":"failover_mds.console.fre906.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_debug_log_pnt_fre904_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.debug_log.fre904.1522562874.log","ReformatBefore_llmount_pnt_stdout":"failover_mds.ReformatBefore_llmount.stdout.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_dmesg_pnt_fre902_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.dmesg.fre902.1522562874.log","stderr":"failover_mds.stderr.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_mount-info_pnt_log":"failover_mds.mount-info.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_dmesg_pnt_fre901_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.dmesg.fre901.1522562874.log","ReformatBefore_llmount_pnt_stderr":"failover_mds.ReformatBefore_llmount.stderr.log","recovery-double-scale_pnt_test_pairwise_fail_pnt_run_dd_debug_pnt_fre906_pnt_log_pnt_fre906":"failover_mds.recovery-double-scale.test_pairwise_fail.run_dd_debug.fre906.log.fre906.log","ReformatBefore_llmountcleanup_pnt_stderr":"failover_mds.ReformatBefore_llmountcleanup.stderr.log","stdout":"failover_mds.stdout.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre906_pnt_log":"failover_mds.messages.fre906.log","_recovery-random-scale_pnt_test_fail_client_mds_pnt_run_dbench_debug_pnt_fre908_pnt_log_pnt_fre908":"failover_mds._recovery-random-scale.test_fail_client_mds.run_dbench_debug.fre908.log.fre908.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_debug_log_pnt_fre902_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.debug_log.fre902.1522562874.log","console_pnt_fre905":"failover_mds.console.fre905.log","ReformatBefore_llmountcleanup_pnt_stdout":"failover_mds.ReformatBefore_llmountcleanup.stdout.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre902_pnt_log":"failover_mds.messages.fre902.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre907_pnt_log":"failover_mds.messages.fre907.log","_recovery-mds-scale_pnt_test_failover_mds_pnt_run_dbench_debug_pnt_fre908_pnt_log_pnt_fre908":"failover_mds._recovery-mds-scale.test_failover_mds.run_dbench_debug.fre908.log.fre908.log","console_pnt_fre907":"failover_mds.console.fre907.log","_recovery-random-scale_pnt_test_fail_client_mds_pnt_run_tar_debug_pnt_fre907_pnt_log_pnt_fre907":"failover_mds._recovery-random-scale.test_fail_client_mds.run_tar_debug.fre907.log.fre907.log","_recovery-mds-scale_pnt_test_failover_mds_pnt_run_dd_debug_pnt_fre906_pnt_log_pnt_fre906":"failover_mds._recovery-mds-scale.test_failover_mds.run_dd_debug.fre906.log.fre906.log","recovery-double-scale_pnt_test_pairwise_fail_pnt_run_tar_stdout_pnt_fre907_pnt_log_pnt_fre907":"failover_mds.recovery-double-scale.test_pairwise_fail.run_tar_stdout.fre907.log.fre907.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_memory-info_pnt_log":"failover_mds.memory-info.log","console_pnt_fre901":"failover_mds.console.fre901.log","_recovery-mds-scale_pnt_test_failover_mds_pnt_run_tar_debug_pnt_fre907_pnt_log_pnt_fre907":"failover_mds._recovery-mds-scale.test_failover_mds.run_tar_debug.fre907.log.fre907.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_dmesg_pnt_fre903_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.dmesg.fre903.1522562874.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_debug_log_pnt_fre901_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.debug_log.fre901.1522562874.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_dmesg_pnt_fre905_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.dmesg.fre905.1522562874.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre903_pnt_log":"failover_mds.messages.fre903.log","recovery-double-scale_pnt_test_pairwise_fail_pnt_run_dbench_debug_pnt_fre908_pnt_log_pnt_fre908":"failover_mds.recovery-double-scale.test_pairwise_fail.run_dbench_debug.fre908.log.fre908.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre908_pnt_log":"failover_mds.messages.fre908.log","console_pnt_fre904":"failover_mds.console.fre904.log","console_pnt_fre902":"failover_mds.console.fre902.log","recovery-double-scale_pnt_test_pairwise_fail_pnt_run_dbench_stdout_pnt_fre908_pnt_log_pnt_fre908":"failover_mds.recovery-double-scale.test_pairwise_fail.run_dbench_stdout.fre908.log.fre908.log","_recovery-random-scale_pnt_test_fail_client_mds_pnt_run_dd_debug_pnt_fre906_pnt_log_pnt_fre906":"failover_mds._recovery-random-scale.test_fail_client_mds.run_dd_debug.fre906.log.fre906.log","recovery-double-scale_pnt_test_pairwise_fail_pnt_run_dd_stdout_pnt_fre906_pnt_log_pnt_fre906":"failover_mds.recovery-double-scale.test_pairwise_fail.run_dd_stdout.fre906.log.fre906.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_dmesg_pnt_fre904_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.dmesg.fre904.1522562874.log","_recovery-mds-scale_pnt_test_failover_mds_pnt_run_tar_stdout_pnt_fre907_pnt_log_pnt_fre907":"failover_mds._recovery-mds-scale.test_failover_mds.run_tar_stdout.fre907.log.fre907.log","console_pnt_fre903":"failover_mds.console.fre903.log","lctllog_pnt_recovery-mds-scale_pnt_test_failover_mds_pnt_debug_log_pnt_fre903_pnt_1522562874":"failover_mds.lctllog.recovery-mds-scale.test_failover_mds.debug_log.fre903.1522562874.log","\/shared\/fremont\/test-results\/xperior-octet\/157\/\/kvm9-octet-1\/xperior-workdir\/recovery-mds-scale\/failover_mds_pnt_messages_pnt_fre905_pnt_log":"failover_mds.messages.fre905.log","recovery-double-scale_pnt_test_pairwise_fail_pnt_run_tar_debug_pnt_fre907_pnt_log_pnt_fre907":"failover_mds.recovery-double-scale.test_pairwise_fail.run_tar_debug.fre907.log.fre907.log"}"4"1522566955"failover_mds""recovery-mds-scale.sh""50300""extropy-kvm9.dev.cray.com""failover_mds""no""data\/schemas\/testds.yaml""failover_mds.console.*.log""invoked oom-killer""LUS-5774""10718""2""10000"0"c17455@cray.com""Elena Gryaznova""recovery-mds-scale test_failover_mds test failed: Oom on client"{"components_0":"Lustre Client"}" failover MDS ""1"null