[ 27.937742] scsi 0:0:131:0: serial_number(ZA1ALLKE0000R8302WN6) [ 27.943663] scsi 0:0:131:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.966913] mpt3sas_cm0: detecting: handle(0x006c), sas_address(0x5000c500954cf9aa), phy(9) [ 27.975267] mpt3sas_cm0: REPORT_LUNS: handle(0x006c), retries(0) [ 27.992231] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006c), lun(0) [ 27.999321] scsi 0:0:132:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.007980] scsi 0:0:132:0: SSP: handle(0x006c), sas_addr(0x5000c500954cf9aa), phy(9), device_name(0x5000c500954cf9a8) [ 28.018662] scsi 0:0:132:0: enclosure logical id(0x5001636001a448fd), slot(9) [ 28.025881] scsi 0:0:132:0: enclosure level(0x0000), connector name( C0 ) [ 28.032775] scsi 0:0:132:0: serial_number(ZA1ALFGA0000R8302NZ9) [ 28.038696] scsi 0:0:132:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.061910] mpt3sas_cm0: detecting: handle(0x006d), sas_address(0x5000c500954cbd1e), phy(10) [ 28.070350] mpt3sas_cm0: REPORT_LUNS: handle(0x006d), retries(0) [ 28.086856] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006d), lun(0) [ 28.093940] scsi 0:0:133:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.102608] scsi 0:0:133:0: SSP: handle(0x006d), sas_addr(0x5000c500954cbd1e), phy(10), device_name(0x5000c500954cbd1c) [ 28.113384] scsi 0:0:133:0: enclosure logical id(0x5001636001a448fd), slot(10) [ 28.120691] scsi 0:0:133:0: enclosure level(0x0000), connector name( C0 ) [ 28.127582] scsi 0:0:133:0: serial_number(ZA1ALLB70000R8301NEA) [ 28.133502] scsi 0:0:133:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.156916] mpt3sas_cm0: detecting: handle(0x006e), sas_address(0x5000c500954dffba), phy(11) [ 28.165358] mpt3sas_cm0: REPORT_LUNS: handle(0x006e), retries(0) [ 28.180168] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006e), lun(0) [ 28.187264] scsi 0:0:134:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.195924] scsi 0:0:134:0: SSP: handle(0x006e), sas_addr(0x5000c500954dffba), phy(11), device_name(0x5000c500954dffb8) [ 28.206700] scsi 0:0:134:0: enclosure logical id(0x5001636001a448fd), slot(11) [ 28.214007] scsi 0:0:134:0: enclosure level(0x0000), connector name( C0 ) [ 28.220900] scsi 0:0:134:0: serial_number(ZA1AM57M0000R8314SAB) [ 28.226819] scsi 0:0:134:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.249917] mpt3sas_cm0: detecting: handle(0x006f), sas_address(0x5000c50095430ade), phy(12) [ 28.258351] mpt3sas_cm0: REPORT_LUNS: handle(0x006f), retries(0) [ 28.277509] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006f), lun(0) [ 28.284601] scsi 0:0:135:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.293262] scsi 0:0:135:0: SSP: handle(0x006f), sas_addr(0x5000c50095430ade), phy(12), device_name(0x5000c50095430adc) [ 28.304038] scsi 0:0:135:0: enclosure logical id(0x5001636001a448fd), slot(12) [ 28.311344] scsi 0:0:135:0: enclosure level(0x0000), connector name( C0 ) [ 28.318237] scsi 0:0:135:0: serial_number(ZA1ACB090000R8301BVU) [ 28.324156] scsi 0:0:135:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.346917] mpt3sas_cm0: detecting: handle(0x0070), sas_address(0x5000c50095432af2), phy(13) [ 28.355353] mpt3sas_cm0: REPORT_LUNS: handle(0x0070), retries(0) [ 28.374256] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0070), lun(0) [ 28.381356] scsi 0:0:136:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.390014] scsi 0:0:136:0: SSP: handle(0x0070), sas_addr(0x5000c50095432af2), phy(13), device_name(0x5000c50095432af0) [ 28.400786] scsi 0:0:136:0: enclosure logical id(0x5001636001a448fd), slot(13) [ 28.408093] scsi 0:0:136:0: enclosure level(0x0000), connector name( C0 ) [ 28.414986] scsi 0:0:136:0: serial_number(ZA1AGGJT0000R703TVW5) [ 28.420906] scsi 0:0:136:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.443921] mpt3sas_cm0: detecting: handle(0x0071), sas_address(0x5000c500954cb72a), phy(14) [ 28.452362] mpt3sas_cm0: REPORT_LUNS: handle(0x0071), retries(0) [ 28.470188] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0071), lun(0) [ 28.477282] scsi 0:0:137:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.485938] scsi 0:0:137:0: SSP: handle(0x0071), sas_addr(0x5000c500954cb72a), phy(14), device_name(0x5000c500954cb728) [ 28.496711] scsi 0:0:137:0: enclosure logical id(0x5001636001a448fd), slot(14) [ 28.504017] scsi 0:0:137:0: enclosure level(0x0000), connector name( C0 ) [ 28.510911] scsi 0:0:137:0: serial_number(ZA1ALH9W0000C8326CMH) [ 28.516831] scsi 0:0:137:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.539925] mpt3sas_cm0: detecting: handle(0x0072), sas_address(0x5000c500954cb18e), phy(15) [ 28.548370] mpt3sas_cm0: REPORT_LUNS: handle(0x0072), retries(0) [ 28.565890] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0072), lun(0) [ 28.572980] scsi 0:0:138:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.581636] scsi 0:0:138:0: SSP: handle(0x0072), sas_addr(0x5000c500954cb18e), phy(15), device_name(0x5000c500954cb18c) [ 28.592412] scsi 0:0:138:0: enclosure logical id(0x5001636001a448fd), slot(15) [ 28.599718] scsi 0:0:138:0: enclosure level(0x0000), connector name( C0 ) [ 28.606611] scsi 0:0:138:0: serial_number(ZA1ALLL00000R8302WFM) [ 28.612530] scsi 0:0:138:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.635924] mpt3sas_cm0: detecting: handle(0x007b), sas_address(0x5000c500954cdf1e), phy(16) [ 28.644368] mpt3sas_cm0: REPORT_LUNS: handle(0x007b), retries(0) [ 28.660668] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007b), lun(0) [ 28.667781] scsi 0:0:139:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.676436] scsi 0:0:139:0: SSP: handle(0x007b), sas_addr(0x5000c500954cdf1e), phy(16), device_name(0x5000c500954cdf1c) [ 28.687210] scsi 0:0:139:0: enclosure logical id(0x5001636001a448fd), slot(16) [ 28.694515] scsi 0:0:139:0: enclosure level(0x0000), connector name( C0 ) [ 28.701410] scsi 0:0:139:0: serial_number(ZA1ALG1K0000R8318P77) [ 28.707328] scsi 0:0:139:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.729932] mpt3sas_cm0: detecting: handle(0x007c), sas_address(0x5000c500954314a2), phy(17) [ 28.738368] mpt3sas_cm0: REPORT_LUNS: handle(0x007c), retries(0) [ 28.752579] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007c), lun(0) [ 28.759654] scsi 0:0:140:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.768314] scsi 0:0:140:0: SSP: handle(0x007c), sas_addr(0x5000c500954314a2), phy(17), device_name(0x5000c500954314a0) [ 28.779089] scsi 0:0:140:0: enclosure logical id(0x5001636001a448fd), slot(17) [ 28.786393] scsi 0:0:140:0: enclosure level(0x0000), connector name( C0 ) [ 28.793288] scsi 0:0:140:0: serial_number(ZA1AB7ZR0000C83152D3) [ 28.799206] scsi 0:0:140:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.821930] mpt3sas_cm0: detecting: handle(0x007d), sas_address(0x5000c500954320f2), phy(18) [ 28.830369] mpt3sas_cm0: REPORT_LUNS: handle(0x007d), retries(0) [ 28.846601] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007d), lun(0) [ 28.853695] scsi 0:0:141:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.862351] scsi 0:0:141:0: SSP: handle(0x007d), sas_addr(0x5000c500954320f2), phy(18), device_name(0x5000c500954320f0) [ 28.873123] scsi 0:0:141:0: enclosure logical id(0x5001636001a448fd), slot(18) [ 28.880430] scsi 0:0:141:0: enclosure level(0x0000), connector name( C0 ) [ 28.887321] scsi 0:0:141:0: serial_number(ZA1AGN1K0000R828927P) [ 28.893242] scsi 0:0:141:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.915936] mpt3sas_cm0: detecting: handle(0x007e), sas_address(0x5000c500954ceef2), phy(19) [ 28.924376] mpt3sas_cm0: REPORT_LUNS: handle(0x007e), retries(0) [ 28.935887] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007e), lun(0) [ 28.943195] scsi 0:0:142:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.951858] scsi 0:0:142:0: SSP: handle(0x007e), sas_addr(0x5000c500954ceef2), phy(19), device_name(0x5000c500954ceef0) [ 28.962626] scsi 0:0:142:0: enclosure logical id(0x5001636001a448fd), slot(19) [ 28.969932] scsi 0:0:142:0: enclosure level(0x0000), connector name( C0 ) [ 28.976827] scsi 0:0:142:0: serial_number(ZA1ALFNS0000R8314R26) [ 28.982746] scsi 0:0:142:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.005935] mpt3sas_cm0: detecting: handle(0x007f), sas_address(0x5000c500954d2cc2), phy(20) [ 29.014373] mpt3sas_cm0: REPORT_LUNS: handle(0x007f), retries(0) [ 29.030659] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007f), lun(0) [ 29.037755] scsi 0:0:143:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.046417] scsi 0:0:143:0: SSP: handle(0x007f), sas_addr(0x5000c500954d2cc2), phy(20), device_name(0x5000c500954d2cc0) [ 29.057191] scsi 0:0:143:0: enclosure logical id(0x5001636001a448fd), slot(20) [ 29.064498] scsi 0:0:143:0: enclosure level(0x0000), connector name( C0 ) [ 29.071390] scsi 0:0:143:0: serial_number(ZA1ALV7M0000R8314SLA) [ 29.077311] scsi 0:0:143:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.099936] mpt3sas_cm0: detecting: handle(0x0080), sas_address(0x5000c500954e02f2), phy(21) [ 29.108375] mpt3sas_cm0: REPORT_LUNS: handle(0x0080), retries(0) [ 29.126209] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0080), lun(0) [ 29.133315] scsi 0:0:144:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.141969] scsi 0:0:144:0: SSP: handle(0x0080), sas_addr(0x5000c500954e02f2), phy(21), device_name(0x5000c500954e02f0) [ 29.152744] scsi 0:0:144:0: enclosure logical id(0x5001636001a448fd), slot(21) [ 29.160051] scsi 0:0:144:0: enclosure level(0x0000), connector name( C0 ) [ 29.166942] scsi 0:0:144:0: serial_number(ZA1AM68M0000R8317LWE) [ 29.172863] scsi 0:0:144:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.196542] mpt3sas_cm0: detecting: handle(0x0081), sas_address(0x5000c50094bb10ea), phy(22) [ 29.204987] mpt3sas_cm0: REPORT_LUNS: handle(0x0081), retries(0) [ 29.217189] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0081), lun(0) [ 29.224287] scsi 0:0:145:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.232946] scsi 0:0:145:0: SSP: handle(0x0081), sas_addr(0x5000c50094bb10ea), phy(22), device_name(0x5000c50094bb10e8) [ 29.243720] scsi 0:0:145:0: enclosure logical id(0x5001636001a448fd), slot(22) [ 29.251027] scsi 0:0:145:0: enclosure level(0x0000), connector name( C0 ) [ 29.257921] scsi 0:0:145:0: serial_number(ZA198PK10000R813M86T) [ 29.263839] scsi 0:0:145:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.286939] mpt3sas_cm0: detecting: handle(0x0082), sas_address(0x5000c5009543158e), phy(23) [ 29.295382] mpt3sas_cm0: REPORT_LUNS: handle(0x0082), retries(0) [ 29.301934] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0082), lun(0) [ 29.309032] scsi 0:0:146:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.317691] scsi 0:0:146:0: SSP: handle(0x0082), sas_addr(0x5000c5009543158e), phy(23), device_name(0x5000c5009543158c) [ 29.328464] scsi 0:0:146:0: enclosure logical id(0x5001636001a448fd), slot(23) [ 29.335771] scsi 0:0:146:0: enclosure level(0x0000), connector name( C0 ) [ 29.342666] scsi 0:0:146:0: serial_number(ZA1AGHB70000R7244YKG) [ 29.348585] scsi 0:0:146:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.371947] mpt3sas_cm0: detecting: handle(0x008b), sas_address(0x5000c500954ce83e), phy(24) [ 29.380386] mpt3sas_cm0: REPORT_LUNS: handle(0x008b), retries(0) [ 29.398922] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008b), lun(0) [ 29.406012] scsi 0:0:147:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.414670] scsi 0:0:147:0: SSP: handle(0x008b), sas_addr(0x5000c500954ce83e), phy(24), device_name(0x5000c500954ce83c) [ 29.425439] scsi 0:0:147:0: enclosure logical id(0x5001636001a448fd), slot(24) [ 29.432746] scsi 0:0:147:0: enclosure level(0x0000), connector name( C0 ) [ 29.439641] scsi 0:0:147:0: serial_number(ZA1ALFTD0000R8314QF8) [ 29.445568] scsi 0:0:147:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.468948] mpt3sas_cm0: detecting: handle(0x008c), sas_address(0x5000c500954cbffe), phy(25) [ 29.477386] mpt3sas_cm0: REPORT_LUNS: handle(0x008c), retries(0) [ 29.498995] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008c), lun(0) [ 29.506109] scsi 0:0:148:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.514773] scsi 0:0:148:0: SSP: handle(0x008c), sas_addr(0x5000c500954cbffe), phy(25), device_name(0x5000c500954cbffc) [ 29.525543] scsi 0:0:148:0: enclosure logical id(0x5001636001a448fd), slot(25) [ 29.532847] scsi 0:0:148:0: enclosure level(0x0000), connector name( C0 ) [ 29.539743] scsi 0:0:148:0: serial_number(ZA1ALL900000R8301Q9K) [ 29.545669] scsi 0:0:148:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.568952] mpt3sas_cm0: detecting: handle(0x008d), sas_address(0x5000c50095431b4a), phy(26) [ 29.577394] mpt3sas_cm0: REPORT_LUNS: handle(0x008d), retries(0) [ 29.589014] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008d), lun(0) [ 29.596110] scsi 0:0:149:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.604772] scsi 0:0:149:0: SSP: handle(0x008d), sas_addr(0x5000c50095431b4a), phy(26), device_name(0x5000c50095431b48) [ 29.615547] scsi 0:0:149:0: enclosure logical id(0x5001636001a448fd), slot(26) [ 29.622854] scsi 0:0:149:0: enclosure level(0x0000), connector name( C0 ) [ 29.629747] scsi 0:0:149:0: serial_number(ZA1AGN600000C8316G12) [ 29.635666] scsi 0:0:149:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.658957] mpt3sas_cm0: detecting: handle(0x008e), sas_address(0x5000c5009543104a), phy(27) [ 29.667399] mpt3sas_cm0: REPORT_LUNS: handle(0x008e), retries(0) [ 29.684603] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008e), lun(0) [ 29.691704] scsi 0:0:150:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.700368] scsi 0:0:150:0: SSP: handle(0x008e), sas_addr(0x5000c5009543104a), phy(27), device_name(0x5000c50095431048) [ 29.711143] scsi 0:0:150:0: enclosure logical id(0x5001636001a448fd), slot(27) [ 29.718448] scsi 0:0:150:0: enclosure level(0x0000), connector name( C0 ) [ 29.725344] scsi 0:0:150:0: serial_number(ZA1AGHK30000R82893N3) [ 29.731260] scsi 0:0:150:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.753956] mpt3sas_cm0: detecting: handle(0x008f), sas_address(0x5000c50095432b76), phy(28) [ 29.762397] mpt3sas_cm0: REPORT_LUNS: handle(0x008f), retries(0) [ 29.779899] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008f), lun(0) [ 29.786991] scsi 0:0:151:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.795644] scsi 0:0:151:0: SSP: handle(0x008f), sas_addr(0x5000c50095432b76), phy(28), device_name(0x5000c50095432b74) [ 29.806419] scsi 0:0:151:0: enclosure logical id(0x5001636001a448fd), slot(28) [ 29.813725] scsi 0:0:151:0: enclosure level(0x0000), connector name( C0 ) [ 29.820618] scsi 0:0:151:0: serial_number(ZA1AGGJD0000C8315359) [ 29.826539] scsi 0:0:151:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.849961] mpt3sas_cm0: detecting: handle(0x0090), sas_address(0x5000c500954dd24e), phy(29) [ 29.858401] mpt3sas_cm0: REPORT_LUNS: handle(0x0090), retries(0) [ 29.864949] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0090), lun(0) [ 29.872038] scsi 0:0:152:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.880701] scsi 0:0:152:0: SSP: handle(0x0090), sas_addr(0x5000c500954dd24e), phy(29), device_name(0x5000c500954dd24c) [ 29.891475] scsi 0:0:152:0: enclosure logical id(0x5001636001a448fd), slot(29) [ 29.898780] scsi 0:0:152:0: enclosure level(0x0000), connector name( C0 ) [ 29.905675] scsi 0:0:152:0: serial_number(ZA1AM6890000R823QFQ5) [ 29.911593] scsi 0:0:152:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.934964] mpt3sas_cm0: detecting: handle(0x0091), sas_address(0x5000c500954dbe1a), phy(30) [ 29.943406] mpt3sas_cm0: REPORT_LUNS: handle(0x0091), retries(0) [ 29.961244] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0091), lun(0) [ 29.968335] scsi 0:0:153:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.976992] scsi 0:0:153:0: SSP: handle(0x0091), sas_addr(0x5000c500954dbe1a), phy(30), device_name(0x5000c500954dbe18) [ 29.987763] scsi 0:0:153:0: enclosure logical id(0x5001636001a448fd), slot(30) [ 29.995071] scsi 0:0:153:0: enclosure level(0x0000), connector name( C0 ) [ 30.001963] scsi 0:0:153:0: serial_number(ZA1AM6EX0000R8318PT9) [ 30.007883] scsi 0:0:153:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.030961] mpt3sas_cm0: detecting: handle(0x0092), sas_address(0x5000c500954cd67e), phy(31) [ 30.039402] mpt3sas_cm0: REPORT_LUNS: handle(0x0092), retries(0) [ 30.045957] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0092), lun(0) [ 30.053048] scsi 0:0:154:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.061701] scsi 0:0:154:0: SSP: handle(0x0092), sas_addr(0x5000c500954cd67e), phy(31), device_name(0x5000c500954cd67c) [ 30.072474] scsi 0:0:154:0: enclosure logical id(0x5001636001a448fd), slot(31) [ 30.079781] scsi 0:0:154:0: enclosure level(0x0000), connector name( C0 ) [ 30.086674] scsi 0:0:154:0: serial_number(ZA1ALKY80000R8302V6C) [ 30.092592] scsi 0:0:154:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.115974] mpt3sas_cm0: detecting: handle(0x009b), sas_address(0x5000c50095430502), phy(32) [ 30.124413] mpt3sas_cm0: REPORT_LUNS: handle(0x009b), retries(0) [ 30.144367] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009b), lun(0) [ 30.151474] scsi 0:0:155:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.160139] scsi 0:0:155:0: SSP: handle(0x009b), sas_addr(0x5000c50095430502), phy(32), device_name(0x5000c50095430500) [ 30.170913] scsi 0:0:155:0: enclosure logical id(0x5001636001a448fd), slot(32) [ 30.178220] scsi 0:0:155:0: enclosure level(0x0000), connector name( C0 ) [ 30.185113] scsi 0:0:155:0: serial_number(ZA1AG3H30000C83152F1) [ 30.191033] scsi 0:0:155:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.213969] mpt3sas_cm0: detecting: handle(0x009c), sas_address(0x5000c50095430dbe), phy(33) [ 30.222410] mpt3sas_cm0: REPORT_LUNS: handle(0x009c), retries(0) [ 30.236892] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009c), lun(0) [ 30.244000] scsi 0:0:156:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.252663] scsi 0:0:156:0: SSP: handle(0x009c), sas_addr(0x5000c50095430dbe), phy(33), device_name(0x5000c50095430dbc) [ 30.263432] scsi 0:0:156:0: enclosure logical id(0x5001636001a448fd), slot(33) [ 30.270739] scsi 0:0:156:0: enclosure level(0x0000), connector name( C0 ) [ 30.277633] scsi 0:0:156:0: serial_number(ZA1AG8BW0000C83153BT) [ 30.283553] scsi 0:0:156:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.306971] mpt3sas_cm0: detecting: handle(0x009d), sas_address(0x5000c500954d3b3e), phy(34) [ 30.315416] mpt3sas_cm0: REPORT_LUNS: handle(0x009d), retries(0) [ 30.336507] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009d), lun(0) [ 30.343600] scsi 0:0:157:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.352259] scsi 0:0:157:0: SSP: handle(0x009d), sas_addr(0x5000c500954d3b3e), phy(34), device_name(0x5000c500954d3b3c) [ 30.363032] scsi 0:0:157:0: enclosure logical id(0x5001636001a448fd), slot(34) [ 30.370337] scsi 0:0:157:0: enclosure level(0x0000), connector name( C0 ) [ 30.377233] scsi 0:0:157:0: serial_number(ZA1ALXTW0000R8314TUG) [ 30.383151] scsi 0:0:157:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.405976] mpt3sas_cm0: detecting: handle(0x009e), sas_address(0x5000c500954cf556), phy(35) [ 30.414418] mpt3sas_cm0: REPORT_LUNS: handle(0x009e), retries(0) [ 30.431552] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009e), lun(0) [ 30.438653] scsi 0:0:158:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.447326] scsi 0:0:158:0: SSP: handle(0x009e), sas_addr(0x5000c500954cf556), phy(35), device_name(0x5000c500954cf554) [ 30.458101] scsi 0:0:158:0: enclosure logical id(0x5001636001a448fd), slot(35) [ 30.465405] scsi 0:0:158:0: enclosure level(0x0000), connector name( C0 ) [ 30.472299] scsi 0:0:158:0: serial_number(ZA1ALFK10000R8314RYN) [ 30.478217] scsi 0:0:158:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.500975] mpt3sas_cm0: detecting: handle(0x009f), sas_address(0x5000c500954ce76e), phy(36) [ 30.509415] mpt3sas_cm0: REPORT_LUNS: handle(0x009f), retries(0) [ 30.529649] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009f), lun(0) [ 30.536741] scsi 0:0:159:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.545410] scsi 0:0:159:0: SSP: handle(0x009f), sas_addr(0x5000c500954ce76e), phy(36), device_name(0x5000c500954ce76c) [ 30.556182] scsi 0:0:159:0: enclosure logical id(0x5001636001a448fd), slot(36) [ 30.563489] scsi 0:0:159:0: enclosure level(0x0000), connector name( C0 ) [ 30.570383] scsi 0:0:159:0: serial_number(ZA1AM03J0000R83026U2) [ 30.576303] scsi 0:0:159:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.598979] mpt3sas_cm0: detecting: handle(0x00a0), sas_address(0x5000c500954cd8c2), phy(37) [ 30.607419] mpt3sas_cm0: REPORT_LUNS: handle(0x00a0), retries(0) [ 30.618430] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a0), lun(0) [ 30.625537] scsi 0:0:160:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.634195] scsi 0:0:160:0: SSP: handle(0x00a0), sas_addr(0x5000c500954cd8c2), phy(37), device_name(0x5000c500954cd8c0) [ 30.644968] scsi 0:0:160:0: enclosure logical id(0x5001636001a448fd), slot(37) [ 30.652272] scsi 0:0:160:0: enclosure level(0x0000), connector name( C0 ) [ 30.659166] scsi 0:0:160:0: serial_number(ZA1ALKWE0000R8301P1U) [ 30.665086] scsi 0:0:160:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.687984] mpt3sas_cm0: detecting: handle(0x00a1), sas_address(0x5000c500954d08be), phy(38) [ 30.696420] mpt3sas_cm0: REPORT_LUNS: handle(0x00a1), retries(0) [ 30.712161] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a1), lun(0) [ 30.719261] scsi 0:0:161:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.727919] scsi 0:0:161:0: SSP: handle(0x00a1), sas_addr(0x5000c500954d08be), phy(38), device_name(0x5000c500954d08bc) [ 30.738690] scsi 0:0:161:0: enclosure logical id(0x5001636001a448fd), slot(38) [ 30.745996] scsi 0:0:161:0: enclosure level(0x0000), connector name( C0 ) [ 30.752889] scsi 0:0:161:0: serial_number(ZA1ALF360000R8314SBC) [ 30.758810] scsi 0:0:161:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.781985] mpt3sas_cm0: detecting: handle(0x00a2), sas_address(0x5000c500954ce876), phy(39) [ 30.790421] mpt3sas_cm0: REPORT_LUNS: handle(0x00a2), retries(0) [ 30.804776] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a2), lun(0) [ 30.811863] scsi 0:0:162:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.820531] scsi 0:0:162:0: SSP: handle(0x00a2), sas_addr(0x5000c500954ce876), phy(39), device_name(0x5000c500954ce874) [ 30.831306] scsi 0:0:162:0: enclosure logical id(0x5001636001a448fd), slot(39) [ 30.838610] scsi 0:0:162:0: enclosure level(0x0000), connector name( C0 ) [ 30.845504] scsi 0:0:162:0: serial_number(ZA1AH42G0000R83028GY) [ 30.851424] scsi 0:0:162:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.874998] mpt3sas_cm0: detecting: handle(0x00ab), sas_address(0x5000c5009543209e), phy(40) [ 30.883434] mpt3sas_cm0: REPORT_LUNS: handle(0x00ab), retries(0) [ 30.897708] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ab), lun(0) [ 30.904814] scsi 0:0:163:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.913481] scsi 0:0:163:0: SSP: handle(0x00ab), sas_addr(0x5000c5009543209e), phy(40), device_name(0x5000c5009543209c) [ 30.924256] scsi 0:0:163:0: enclosure logical id(0x5001636001a448fd), slot(40) [ 30.931563] scsi 0:0:163:0: enclosure level(0x0000), connector name( C0 ) [ 30.938457] scsi 0:0:163:0: serial_number(ZA1AGGZ80000C8316FAR) [ 30.944376] scsi 0:0:163:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.966988] mpt3sas_cm0: detecting: handle(0x00ac), sas_address(0x5000c50095432532), phy(41) [ 30.975424] mpt3sas_cm0: REPORT_LUNS: handle(0x00ac), retries(0) [ 30.993649] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ac), lun(0) [ 31.000760] scsi 0:0:164:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.009424] scsi 0:0:164:0: SSP: handle(0x00ac), sas_addr(0x5000c50095432532), phy(41), device_name(0x5000c50095432530) [ 31.020199] scsi 0:0:164:0: enclosure logical id(0x5001636001a448fd), slot(41) [ 31.027505] scsi 0:0:164:0: enclosure level(0x0000), connector name( C0 ) [ 31.034399] scsi 0:0:164:0: serial_number(ZA1AGGSZ0000C8316FP5) [ 31.040319] scsi 0:0:164:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.062988] mpt3sas_cm0: detecting: handle(0x00ad), sas_address(0x5000c5009544627a), phy(42) [ 31.071430] mpt3sas_cm0: REPORT_LUNS: handle(0x00ad), retries(0) [ 31.088345] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ad), lun(0) [ 31.095488] scsi 0:0:165:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.104151] scsi 0:0:165:0: SSP: handle(0x00ad), sas_addr(0x5000c5009544627a), phy(42), device_name(0x5000c50095446278) [ 31.114920] scsi 0:0:165:0: enclosure logical id(0x5001636001a448fd), slot(42) [ 31.122227] scsi 0:0:165:0: enclosure level(0x0000), connector name( C0 ) [ 31.129119] scsi 0:0:165:0: serial_number(ZA1AGTHC0000R8289DMD) [ 31.135038] scsi 0:0:165:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.157993] mpt3sas_cm0: detecting: handle(0x00ae), sas_address(0x5000c50095431fea), phy(43) [ 31.166434] mpt3sas_cm0: REPORT_LUNS: handle(0x00ae), retries(0) [ 31.178434] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ae), lun(0) [ 31.229996] scsi 0:0:166:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.238647] scsi 0:0:166:0: SSP: handle(0x00ae), sas_addr(0x5000c50095431fea), phy(43), device_name(0x5000c50095431fe8) [ 31.249423] scsi 0:0:166:0: enclosure logical id(0x5001636001a448fd), slot(43) [ 31.256727] scsi 0:0:166:0: enclosure level(0x0000), connector name( C0 ) [ 31.263624] scsi 0:0:166:0: serial_number(ZA1AGN2P0000R829D74G) [ 31.269550] scsi 0:0:166:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.293604] mpt3sas_cm0: detecting: handle(0x00af), sas_address(0x5000c50095431a0e), phy(44) [ 31.302046] mpt3sas_cm0: REPORT_LUNS: handle(0x00af), retries(0) [ 31.314834] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00af), lun(0) [ 31.321934] scsi 0:0:167:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.330584] scsi 0:0:167:0: SSP: handle(0x00af), sas_addr(0x5000c50095431a0e), phy(44), device_name(0x5000c50095431a0c) [ 31.341361] scsi 0:0:167:0: enclosure logical id(0x5001636001a448fd), slot(44) [ 31.348667] scsi 0:0:167:0: enclosure level(0x0000), connector name( C0 ) [ 31.355561] scsi 0:0:167:0: serial_number(ZA1AGN800000C8316FZ6) [ 31.361478] scsi 0:0:167:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.385003] mpt3sas_cm0: detecting: handle(0x00b0), sas_address(0x5000c50095430d5e), phy(45) [ 31.393447] mpt3sas_cm0: REPORT_LUNS: handle(0x00b0), retries(0) [ 31.405551] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b0), lun(0) [ 31.412653] scsi 0:0:168:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.421315] scsi 0:0:168:0: SSP: handle(0x00b0), sas_addr(0x5000c50095430d5e), phy(45), device_name(0x5000c50095430d5c) [ 31.432087] scsi 0:0:168:0: enclosure logical id(0x5001636001a448fd), slot(45) [ 31.439390] scsi 0:0:168:0: enclosure level(0x0000), connector name( C0 ) [ 31.446285] scsi 0:0:168:0: serial_number(ZA1AG8C40000C83153SW) [ 31.452203] scsi 0:0:168:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.475001] mpt3sas_cm0: detecting: handle(0x00b1), sas_address(0x5000c5009543220e), phy(46) [ 31.483443] mpt3sas_cm0: REPORT_LUNS: handle(0x00b1), retries(0) [ 31.498513] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b1), lun(0) [ 31.505615] scsi 0:0:169:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.514267] scsi 0:0:169:0: SSP: handle(0x00b1), sas_addr(0x5000c5009543220e), phy(46), device_name(0x5000c5009543220c) [ 31.525039] scsi 0:0:169:0: enclosure logical id(0x5001636001a448fd), slot(46) [ 31.532345] scsi 0:0:169:0: enclosure level(0x0000), connector name( C0 ) [ 31.539237] scsi 0:0:169:0: serial_number(ZA1AGH6M0000R7354S3K) [ 31.545157] scsi 0:0:169:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.568003] mpt3sas_cm0: detecting: handle(0x00b2), sas_address(0x5000c50095430eca), phy(47) [ 31.576439] mpt3sas_cm0: REPORT_LUNS: handle(0x00b2), retries(0) [ 31.590344] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b2), lun(0) [ 31.597443] scsi 0:0:170:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.606102] scsi 0:0:170:0: SSP: handle(0x00b2), sas_addr(0x5000c50095430eca), phy(47), device_name(0x5000c50095430ec8) [ 31.616872] scsi 0:0:170:0: enclosure logical id(0x5001636001a448fd), slot(47) [ 31.624177] scsi 0:0:170:0: enclosure level(0x0000), connector name( C0 ) [ 31.631071] scsi 0:0:170:0: serial_number(ZA1AGHK50000R829CUPQ) [ 31.636992] scsi 0:0:170:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.660000] mpt3sas_cm0: detecting: handle(0x00bb), sas_address(0x5000c500954cd14a), phy(48) [ 31.668441] mpt3sas_cm0: REPORT_LUNS: handle(0x00bb), retries(0) [ 31.682343] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bb), lun(0) [ 31.689448] scsi 0:0:171:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.698110] scsi 0:0:171:0: SSP: handle(0x00bb), sas_addr(0x5000c500954cd14a), phy(48), device_name(0x5000c500954cd148) [ 31.708881] scsi 0:0:171:0: enclosure logical id(0x5001636001a448fd), slot(48) [ 31.716186] scsi 0:0:171:0: enclosure level(0x0000), connector name( C0 ) [ 31.723080] scsi 0:0:171:0: serial_number(ZA1ALGGL0000R8303FE1) [ 31.728999] scsi 0:0:171:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.752009] mpt3sas_cm0: detecting: handle(0x00bc), sas_address(0x5000c500954d432a), phy(49) [ 31.760447] mpt3sas_cm0: REPORT_LUNS: handle(0x00bc), retries(0) [ 31.777752] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bc), lun(0) [ 31.784850] scsi 0:0:172:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.793501] scsi 0:0:172:0: SSP: handle(0x00bc), sas_addr(0x5000c500954d432a), phy(49), device_name(0x5000c500954d4328) [ 31.804276] scsi 0:0:172:0: enclosure logical id(0x5001636001a448fd), slot(49) [ 31.811582] scsi 0:0:172:0: enclosure level(0x0000), connector name( C0 ) [ 31.818477] scsi 0:0:172:0: serial_number(ZA1ADGNZ0000R8318PG2) [ 31.824394] scsi 0:0:172:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.847012] mpt3sas_cm0: detecting: handle(0x00bd), sas_address(0x5000c500954ce106), phy(50) [ 31.855454] mpt3sas_cm0: REPORT_LUNS: handle(0x00bd), retries(0) [ 31.872715] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bd), lun(0) [ 31.879798] scsi 0:0:173:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.888462] scsi 0:0:173:0: SSP: handle(0x00bd), sas_addr(0x5000c500954ce106), phy(50), device_name(0x5000c500954ce104) [ 31.899231] scsi 0:0:173:0: enclosure logical id(0x5001636001a448fd), slot(50) [ 31.906537] scsi 0:0:173:0: enclosure level(0x0000), connector name( C0 ) [ 31.913431] scsi 0:0:173:0: serial_number(ZA1ALKK10000R8301BA6) [ 31.919351] scsi 0:0:173:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.945015] mpt3sas_cm0: detecting: handle(0x00be), sas_address(0x5000c500954d38ae), phy(51) [ 31.953457] mpt3sas_cm0: REPORT_LUNS: handle(0x00be), retries(0) [ 31.965439] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00be), lun(0) [ 31.972540] scsi 0:0:174:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.981197] scsi 0:0:174:0: SSP: handle(0x00be), sas_addr(0x5000c500954d38ae), phy(51), device_name(0x5000c500954d38ac) [ 31.991968] scsi 0:0:174:0: enclosure logical id(0x5001636001a448fd), slot(51) [ 31.999273] scsi 0:0:174:0: enclosure level(0x0000), connector name( C0 ) [ 32.006166] scsi 0:0:174:0: serial_number(ZA1ALTXK0000R8303HMP) [ 32.012085] scsi 0:0:174:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.035012] mpt3sas_cm0: detecting: handle(0x00bf), sas_address(0x5000c500954ce652), phy(52) [ 32.043455] mpt3sas_cm0: REPORT_LUNS: handle(0x00bf), retries(0) [ 32.063576] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bf), lun(0) [ 32.070678] scsi 0:0:175:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.079339] scsi 0:0:175:0: SSP: handle(0x00bf), sas_addr(0x5000c500954ce652), phy(52), device_name(0x5000c500954ce650) [ 32.090112] scsi 0:0:175:0: enclosure logical id(0x5001636001a448fd), slot(52) [ 32.097416] scsi 0:0:175:0: enclosure level(0x0000), connector name( C0 ) [ 32.104310] scsi 0:0:175:0: serial_number(ZA1ALFW80000R8314Q46) [ 32.110229] scsi 0:0:175:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.133016] mpt3sas_cm0: detecting: handle(0x00c0), sas_address(0x5000c500954cb542), phy(53) [ 32.141452] mpt3sas_cm0: REPORT_LUNS: handle(0x00c0), retries(0) [ 32.161871] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c0), lun(0) [ 32.168968] scsi 0:0:176:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.177634] scsi 0:0:176:0: SSP: handle(0x00c0), sas_addr(0x5000c500954cb542), phy(53), device_name(0x5000c500954cb540) [ 32.188410] scsi 0:0:176:0: enclosure logical id(0x5001636001a448fd), slot(53) [ 32.195716] scsi 0:0:176:0: enclosure level(0x0000), connector name( C0 ) [ 32.202613] scsi 0:0:176:0: serial_number(ZA1ALH9B0000C8326GM2) [ 32.208537] scsi 0:0:176:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.232021] mpt3sas_cm0: detecting: handle(0x00c1), sas_address(0x5000c500954cef5e), phy(54) [ 32.240463] mpt3sas_cm0: REPORT_LUNS: handle(0x00c1), retries(0) [ 32.257485] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c1), lun(0) [ 32.264591] scsi 0:0:177:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.273249] scsi 0:0:177:0: SSP: handle(0x00c1), sas_addr(0x5000c500954cef5e), phy(54), device_name(0x5000c500954cef5c) [ 32.284023] scsi 0:0:177:0: enclosure logical id(0x5001636001a448fd), slot(54) [ 32.291330] scsi 0:0:177:0: enclosure level(0x0000), connector name( C0 ) [ 32.298222] scsi 0:0:177:0: serial_number(ZA1ALFRE0000R8314RDP) [ 32.304144] scsi 0:0:177:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.330027] mpt3sas_cm0: detecting: handle(0x00c2), sas_address(0x5000c5009543146a), phy(55) [ 32.338468] mpt3sas_cm0: REPORT_LUNS: handle(0x00c2), retries(0) [ 32.358945] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c2), lun(0) [ 32.366041] scsi 0:0:178:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.374695] scsi 0:0:178:0: SSP: handle(0x00c2), sas_addr(0x5000c5009543146a), phy(55), device_name(0x5000c50095431468) [ 32.385470] scsi 0:0:178:0: enclosure logical id(0x5001636001a448fd), slot(55) [ 32.392777] scsi 0:0:178:0: enclosure level(0x0000), connector name( C0 ) [ 32.399670] scsi 0:0:178:0: serial_number(ZA1AD8AT0000C8315377) [ 32.405589] scsi 0:0:178:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.429027] mpt3sas_cm0: detecting: handle(0x00c8), sas_address(0x5000c500954d020e), phy(56) [ 32.437468] mpt3sas_cm0: REPORT_LUNS: handle(0x00c8), retries(0) [ 32.455281] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c8), lun(0) [ 32.462390] scsi 0:0:179:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.471049] scsi 0:0:179:0: SSP: handle(0x00c8), sas_addr(0x5000c500954d020e), phy(56), device_name(0x5000c500954d020c) [ 32.481821] scsi 0:0:179:0: enclosure logical id(0x5001636001a448fd), slot(56) [ 32.489125] scsi 0:0:179:0: enclosure level(0x0000), connector name( C0 ) [ 32.496020] scsi 0:0:179:0: serial_number(ZA1ALF9M0000R8314R9F) [ 32.501940] scsi 0:0:179:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.525026] mpt3sas_cm0: detecting: handle(0x00c9), sas_address(0x5000c500954d0af2), phy(57) [ 32.533464] mpt3sas_cm0: REPORT_LUNS: handle(0x00c9), retries(0) [ 32.547280] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c9), lun(0) [ 32.554394] scsi 0:0:180:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.563064] scsi 0:0:180:0: SSP: handle(0x00c9), sas_addr(0x5000c500954d0af2), phy(57), device_name(0x5000c500954d0af0) [ 32.573836] scsi 0:0:180:0: enclosure logical id(0x5001636001a448fd), slot(57) [ 32.581142] scsi 0:0:180:0: enclosure level(0x0000), connector name( C0 ) [ 32.588036] scsi 0:0:180:0: serial_number(ZA1AFT510000R8314QQS) [ 32.593954] scsi 0:0:180:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.617027] mpt3sas_cm0: detecting: handle(0x00ca), sas_address(0x5000c500954cd682), phy(58) [ 32.625463] mpt3sas_cm0: REPORT_LUNS: handle(0x00ca), retries(0) [ 32.639274] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ca), lun(0) [ 32.646375] scsi 0:0:181:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.655040] scsi 0:0:181:0: SSP: handle(0x00ca), sas_addr(0x5000c500954cd682), phy(58), device_name(0x5000c500954cd680) [ 32.665810] scsi 0:0:181:0: enclosure logical id(0x5001636001a448fd), slot(58) [ 32.673116] scsi 0:0:181:0: enclosure level(0x0000), connector name( C0 ) [ 32.680008] scsi 0:0:181:0: serial_number(ZA1AJGTA0000R8302VY7) [ 32.685929] scsi 0:0:181:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.709034] mpt3sas_cm0: detecting: handle(0x00cb), sas_address(0x5000c500954cd34e), phy(59) [ 32.717471] mpt3sas_cm0: REPORT_LUNS: handle(0x00cb), retries(0) [ 32.733565] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00cb), lun(0) [ 32.740677] scsi 0:0:182:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.749333] scsi 0:0:182:0: SSP: handle(0x00cb), sas_addr(0x5000c500954cd34e), phy(59), device_name(0x5000c500954cd34c) [ 32.760106] scsi 0:0:182:0: enclosure logical id(0x5001636001a448fd), slot(59) [ 32.767413] scsi 0:0:182:0: enclosure level(0x0000), connector name( C0 ) [ 32.774305] scsi 0:0:182:0: serial_number(ZA1ALL000000R8302TYC) [ 32.780224] scsi 0:0:182:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.804214] mpt3sas_cm0: detecting: handle(0x00ce), sas_address(0x5001636001a448fd), phy(76) [ 32.812666] mpt3sas_cm0: REPORT_LUNS: handle(0x00ce), retries(0) [ 32.818856] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ce), lun(0) [ 32.825333] scsi 0:0:183:0: Enclosure QCT JB4602 SIM 1 1500 PQ: 0 ANSI: 6 [ 32.833797] scsi 0:0:183:0: set ignore_delay_remove for handle(0x00ce) [ 32.840326] scsi 0:0:183:0: SES: handle(0x00ce), sas_addr(0x5001636001a448fd), phy(76), device_name(0x5001636001a448fd) [ 32.851099] scsi 0:0:183:0: enclosure logical id(0x5001636001a448fd), slot(60) [ 32.858405] scsi 0:0:183:0: enclosure level(0x0000), connector name( C0 ) [ 32.865297] scsi 0:0:183:0: serial_number(EO0153600219 ) [ 32.870871] scsi 0:0:183:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.892333] mpt3sas_cm0: expander_add: handle(0x00cf), parent(0x0002), sas_addr(0x50016360020accbf), phys(79) [ 32.914046] mpt3sas_cm0: detecting: handle(0x00d0), sas_address(0x5000c500954dc5be), phy(0) [ 32.922398] mpt3sas_cm0: REPORT_LUNS: handle(0x00d0), retries(0) [ 32.942022] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d0), lun(0) [ 32.949133] scsi 0:0:184:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.957796] scsi 0:0:184:0: SSP: handle(0x00d0), sas_addr(0x5000c500954dc5be), phy(0), device_name(0x5000c500954dc5bc) [ 32.968483] scsi 0:0:184:0: enclosure logical id(0x50016360020accbd), slot(0) [ 32.975705] scsi 0:0:184:0: enclosure level(0x0000), connector name( C2 ) [ 32.982598] scsi 0:0:184:0: serial_number(ZA1AM2YX0000R8314U1K) [ 32.988515] scsi 0:0:184:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.012043] mpt3sas_cm0: detecting: handle(0x00d1), sas_address(0x5000c500954cb1ca), phy(1) [ 33.020396] mpt3sas_cm0: REPORT_LUNS: handle(0x00d1), retries(0) [ 33.042178] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d1), lun(0) [ 33.049277] scsi 0:0:185:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.057942] scsi 0:0:185:0: SSP: handle(0x00d1), sas_addr(0x5000c500954cb1ca), phy(1), device_name(0x5000c500954cb1c8) [ 33.068630] scsi 0:0:185:0: enclosure logical id(0x50016360020accbd), slot(1) [ 33.075849] scsi 0:0:185:0: enclosure level(0x0000), connector name( C2 ) [ 33.082743] scsi 0:0:185:0: serial_number(ZA1ALLK80000R8302WHW) [ 33.088663] scsi 0:0:185:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.112046] mpt3sas_cm0: detecting: handle(0x00d2), sas_address(0x5000c500954d8866), phy(2) [ 33.120394] mpt3sas_cm0: REPORT_LUNS: handle(0x00d2), retries(0) [ 33.133561] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d2), lun(0) [ 33.140685] scsi 0:0:186:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.149343] scsi 0:0:186:0: SSP: handle(0x00d2), sas_addr(0x5000c500954d8866), phy(2), device_name(0x5000c500954d8864) [ 33.160031] scsi 0:0:186:0: enclosure logical id(0x50016360020accbd), slot(2) [ 33.167250] scsi 0:0:186:0: enclosure level(0x0000), connector name( C2 ) [ 33.174144] scsi 0:0:186:0: serial_number(ZA1AM3ZV0000R8301RFQ) [ 33.180064] scsi 0:0:186:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.204045] mpt3sas_cm0: detecting: handle(0x00d3), sas_address(0x5000c500954cbdba), phy(3) [ 33.212396] mpt3sas_cm0: REPORT_LUNS: handle(0x00d3), retries(0) [ 33.231191] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d3), lun(0) [ 33.238319] scsi 0:0:187:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.246977] scsi 0:0:187:0: SSP: handle(0x00d3), sas_addr(0x5000c500954cbdba), phy(3), device_name(0x5000c500954cbdb8) [ 33.257664] scsi 0:0:187:0: enclosure logical id(0x50016360020accbd), slot(3) [ 33.264883] scsi 0:0:187:0: enclosure level(0x0000), connector name( C2 ) [ 33.271776] scsi 0:0:187:0: serial_number(ZA1ALLAN0000R8301RHF) [ 33.277695] scsi 0:0:187:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.301043] mpt3sas_cm0: detecting: handle(0x00d4), sas_address(0x5000c500954d882e), phy(4) [ 33.309394] mpt3sas_cm0: REPORT_LUNS: handle(0x00d4), retries(0) [ 33.330194] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d4), lun(0) [ 33.337311] scsi 0:0:188:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.345969] scsi 0:0:188:0: SSP: handle(0x00d4), sas_addr(0x5000c500954d882e), phy(4), device_name(0x5000c500954d882c) [ 33.356656] scsi 0:0:188:0: enclosure logical id(0x50016360020accbd), slot(4) [ 33.363876] scsi 0:0:188:0: enclosure level(0x0000), connector name( C2 ) [ 33.370770] scsi 0:0:188:0: serial_number(ZA1AM40C0000R8302N84) [ 33.376690] scsi 0:0:188:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.400048] mpt3sas_cm0: detecting: handle(0x00d5), sas_address(0x5000c500954d8efa), phy(5) [ 33.408405] mpt3sas_cm0: REPORT_LUNS: handle(0x00d5), retries(0) [ 33.429038] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d5), lun(0) [ 33.436150] scsi 0:0:189:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.444809] scsi 0:0:189:0: SSP: handle(0x00d5), sas_addr(0x5000c500954d8efa), phy(5), device_name(0x5000c500954d8ef8) [ 33.455494] scsi 0:0:189:0: enclosure logical id(0x50016360020accbd), slot(5) [ 33.462715] scsi 0:0:189:0: enclosure level(0x0000), connector name( C2 ) [ 33.469609] scsi 0:0:189:0: serial_number(ZA1A8EY30000R8314UMS) [ 33.475534] scsi 0:0:189:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.499052] mpt3sas_cm0: detecting: handle(0x00d6), sas_address(0x5000c500954d8c2a), phy(6) [ 33.507406] mpt3sas_cm0: REPORT_LUNS: handle(0x00d6), retries(0) [ 33.518523] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d6), lun(0) [ 33.525619] scsi 0:0:190:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.534277] scsi 0:0:190:0: SSP: handle(0x00d6), sas_addr(0x5000c500954d8c2a), phy(6), device_name(0x5000c500954d8c28) [ 33.544961] scsi 0:0:190:0: enclosure logical id(0x50016360020accbd), slot(6) [ 33.552182] scsi 0:0:190:0: enclosure level(0x0000), connector name( C2 ) [ 33.559079] scsi 0:0:190:0: serial_number(ZA1AM7PV0000R8317PXS) [ 33.565004] scsi 0:0:190:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.588058] mpt3sas_cm0: detecting: handle(0x00d7), sas_address(0x5000c500954cda8a), phy(7) [ 33.596407] mpt3sas_cm0: REPORT_LUNS: handle(0x00d7), retries(0) [ 33.608200] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d7), lun(0) [ 33.615315] scsi 0:0:191:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.623976] scsi 0:0:191:0: SSP: handle(0x00d7), sas_addr(0x5000c500954cda8a), phy(7), device_name(0x5000c500954cda88) [ 33.634665] scsi 0:0:191:0: enclosure logical id(0x50016360020accbd), slot(7) [ 33.641884] scsi 0:0:191:0: enclosure level(0x0000), connector name( C2 ) [ 33.648775] scsi 0:0:191:0: serial_number(ZA1ALG5T0000R8317LT0) [ 33.654696] scsi 0:0:191:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.678060] mpt3sas_cm0: detecting: handle(0x00d8), sas_address(0x5000c500954d377e), phy(8) [ 33.686411] mpt3sas_cm0: REPORT_LUNS: handle(0x00d8), retries(0) [ 33.707620] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d8), lun(0) [ 33.714741] scsi 0:0:192:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.723401] scsi 0:0:192:0: SSP: handle(0x00d8), sas_addr(0x5000c500954d377e), phy(8), device_name(0x5000c500954d377c) [ 33.734083] scsi 0:0:192:0: enclosure logical id(0x50016360020accbd), slot(8) [ 33.741301] scsi 0:0:192:0: enclosure level(0x0000), connector name( C2 ) [ 33.748195] scsi 0:0:192:0: serial_number(ZA1ALXXS0000R83027F8) [ 33.754114] scsi 0:0:192:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.777056] mpt3sas_cm0: detecting: handle(0x00d9), sas_address(0x5000c500954d42a6), phy(9) [ 33.785407] mpt3sas_cm0: REPORT_LUNS: handle(0x00d9), retries(0) [ 33.798875] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d9), lun(0) [ 33.843912] scsi 0:0:193:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.852550] scsi 0:0:193:0: SSP: handle(0x00d9), sas_addr(0x5000c500954d42a6), phy(9), device_name(0x5000c500954d42a4) [ 33.863236] scsi 0:0:193:0: enclosure logical id(0x50016360020accbd), slot(9) [ 33.870457] scsi 0:0:193:0: enclosure level(0x0000), connector name( C2 ) [ 33.877349] scsi 0:0:193:0: serial_number(ZA1ALXKS0000R8302UX2) [ 33.883270] scsi 0:0:193:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.906064] mpt3sas_cm0: detecting: handle(0x00da), sas_address(0x5000c500954de29a), phy(10) [ 33.914499] mpt3sas_cm0: REPORT_LUNS: handle(0x00da), retries(0) [ 33.936097] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00da), lun(0) [ 33.943203] scsi 0:0:194:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.951869] scsi 0:0:194:0: SSP: handle(0x00da), sas_addr(0x5000c500954de29a), phy(10), device_name(0x5000c500954de298) [ 33.962647] scsi 0:0:194:0: enclosure logical id(0x50016360020accbd), slot(10) [ 33.969953] scsi 0:0:194:0: enclosure level(0x0000), connector name( C2 ) [ 33.976847] scsi 0:0:194:0: serial_number(ZA1AM2EF0000R83170V4) [ 33.982766] scsi 0:0:194:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.006070] mpt3sas_cm0: detecting: handle(0x00db), sas_address(0x5000c500954deea6), phy(11) [ 34.014507] mpt3sas_cm0: REPORT_LUNS: handle(0x00db), retries(0) [ 34.027706] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00db), lun(0) [ 34.034807] scsi 0:0:195:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.043469] scsi 0:0:195:0: SSP: handle(0x00db), sas_addr(0x5000c500954deea6), phy(11), device_name(0x5000c500954deea4) [ 34.054237] scsi 0:0:195:0: enclosure logical id(0x50016360020accbd), slot(11) [ 34.061543] scsi 0:0:195:0: enclosure level(0x0000), connector name( C2 ) [ 34.068438] scsi 0:0:195:0: serial_number(ZA1AM28R0000R828951S) [ 34.074357] scsi 0:0:195:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.097066] mpt3sas_cm0: detecting: handle(0x00dc), sas_address(0x5000c500954d3c16), phy(12) [ 34.105509] mpt3sas_cm0: REPORT_LUNS: handle(0x00dc), retries(0) [ 34.123425] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00dc), lun(0) [ 34.130551] scsi 0:0:196:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.139202] scsi 0:0:196:0: SSP: handle(0x00dc), sas_addr(0x5000c500954d3c16), phy(12), device_name(0x5000c500954d3c14) [ 34.149972] scsi 0:0:196:0: enclosure logical id(0x50016360020accbd), slot(12) [ 34.157280] scsi 0:0:196:0: enclosure level(0x0000), connector name( C2 ) [ 34.164171] scsi 0:0:196:0: serial_number(ZA1ALTM40000R8302PRY) [ 34.170092] scsi 0:0:196:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.193071] mpt3sas_cm0: detecting: handle(0x00dd), sas_address(0x5000c500954da7a2), phy(13) [ 34.201512] mpt3sas_cm0: REPORT_LUNS: handle(0x00dd), retries(0) [ 34.216928] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00dd), lun(0) [ 34.224038] scsi 0:0:197:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.232701] scsi 0:0:197:0: SSP: handle(0x00dd), sas_addr(0x5000c500954da7a2), phy(13), device_name(0x5000c500954da7a0) [ 34.243471] scsi 0:0:197:0: enclosure logical id(0x50016360020accbd), slot(13) [ 34.250778] scsi 0:0:197:0: enclosure level(0x0000), connector name( C2 ) [ 34.257672] scsi 0:0:197:0: serial_number(ZA1AM3FC0000R8303GY3) [ 34.263591] scsi 0:0:197:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.287076] mpt3sas_cm0: detecting: handle(0x00de), sas_address(0x5000c500954cbb62), phy(14) [ 34.295515] mpt3sas_cm0: REPORT_LUNS: handle(0x00de), retries(0) [ 34.307217] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00de), lun(0) [ 34.314329] scsi 0:0:198:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.322995] scsi 0:0:198:0: SSP: handle(0x00de), sas_addr(0x5000c500954cbb62), phy(14), device_name(0x5000c500954cbb60) [ 34.333770] scsi 0:0:198:0: enclosure logical id(0x50016360020accbd), slot(14) [ 34.341077] scsi 0:0:198:0: enclosure level(0x0000), connector name( C2 ) [ 34.347969] scsi 0:0:198:0: serial_number(ZA1ALLBN0000R8301P6N) [ 34.353889] scsi 0:0:198:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.377073] mpt3sas_cm0: detecting: handle(0x00df), sas_address(0x5000c500954dddd2), phy(15) [ 34.385511] mpt3sas_cm0: REPORT_LUNS: handle(0x00df), retries(0) [ 34.404188] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00df), lun(0) [ 34.443511] scsi 0:0:199:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.452167] scsi 0:0:199:0: SSP: handle(0x00df), sas_addr(0x5000c500954dddd2), phy(15), device_name(0x5000c500954dddd0) [ 34.462944] scsi 0:0:199:0: enclosure logical id(0x50016360020accbd), slot(15) [ 34.470248] scsi 0:0:199:0: enclosure level(0x0000), connector name( C2 ) [ 34.477142] scsi 0:0:199:0: serial_number(ZA1AM5TL0000R8314T3E) [ 34.483062] scsi 0:0:199:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.506079] mpt3sas_cm0: detecting: handle(0x00e0), sas_address(0x5000c500954df44e), phy(16) [ 34.514517] mpt3sas_cm0: REPORT_LUNS: handle(0x00e0), retries(0) [ 34.530279] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e0), lun(0) [ 34.537388] scsi 0:0:200:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.546057] scsi 0:0:200:0: SSP: handle(0x00e0), sas_addr(0x5000c500954df44e), phy(16), device_name(0x5000c500954df44c) [ 34.556833] scsi 0:0:200:0: enclosure logical id(0x50016360020accbd), slot(16) [ 34.564137] scsi 0:0:200:0: enclosure level(0x0000), connector name( C2 ) [ 34.571035] scsi 0:0:200:0: serial_number(ZA1AM5E40000R8317LFH) [ 34.576960] scsi 0:0:200:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.600086] mpt3sas_cm0: detecting: handle(0x00e1), sas_address(0x5000c500954cbb3e), phy(17) [ 34.608528] mpt3sas_cm0: REPORT_LUNS: handle(0x00e1), retries(0) [ 34.629437] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e1), lun(0) [ 34.636553] scsi 0:0:201:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.645209] scsi 0:0:201:0: SSP: handle(0x00e1), sas_addr(0x5000c500954cbb3e), phy(17), device_name(0x5000c500954cbb3c) [ 34.655981] scsi 0:0:201:0: enclosure logical id(0x50016360020accbd), slot(17) [ 34.663286] scsi 0:0:201:0: enclosure level(0x0000), connector name( C2 ) [ 34.670179] scsi 0:0:201:0: serial_number(ZA1ALH3Z0000C8325Y02) [ 34.676100] scsi 0:0:201:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.699084] mpt3sas_cm0: detecting: handle(0x00e2), sas_address(0x5000c500954cd086), phy(18) [ 34.707520] mpt3sas_cm0: REPORT_LUNS: handle(0x00e2), retries(0) [ 34.727617] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e2), lun(0) [ 34.734732] scsi 0:0:202:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.743386] scsi 0:0:202:0: SSP: handle(0x00e2), sas_addr(0x5000c500954cd086), phy(18), device_name(0x5000c500954cd084) [ 34.754159] scsi 0:0:202:0: enclosure logical id(0x50016360020accbd), slot(18) [ 34.761464] scsi 0:0:202:0: enclosure level(0x0000), connector name( C2 ) [ 34.768360] scsi 0:0:202:0: serial_number(ZA1ALGHH0000R8303G4B) [ 34.774278] scsi 0:0:202:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.797084] mpt3sas_cm0: detecting: handle(0x00e3), sas_address(0x5000c500954d3d22), phy(19) [ 34.805527] mpt3sas_cm0: REPORT_LUNS: handle(0x00e3), retries(0) [ 34.824950] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e3), lun(0) [ 34.832058] scsi 0:0:203:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.840720] scsi 0:0:203:0: SSP: handle(0x00e3), sas_addr(0x5000c500954d3d22), phy(19), device_name(0x5000c500954d3d20) [ 34.851488] scsi 0:0:203:0: enclosure logical id(0x50016360020accbd), slot(19) [ 34.858795] scsi 0:0:203:0: enclosure level(0x0000), connector name( C2 ) [ 34.865688] scsi 0:0:203:0: serial_number(ZA1ALTL50000C8321SL6) [ 34.871607] scsi 0:0:203:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.895090] mpt3sas_cm0: detecting: handle(0x00e4), sas_address(0x5000c500954dbb32), phy(20) [ 34.903531] mpt3sas_cm0: REPORT_LUNS: handle(0x00e4), retries(0) [ 34.920703] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e4), lun(0) [ 34.927811] scsi 0:0:204:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.936472] scsi 0:0:204:0: SSP: handle(0x00e4), sas_addr(0x5000c500954dbb32), phy(20), device_name(0x5000c500954dbb30) [ 34.947242] scsi 0:0:204:0: enclosure logical id(0x50016360020accbd), slot(20) [ 34.954545] scsi 0:0:204:0: enclosure level(0x0000), connector name( C2 ) [ 34.961440] scsi 0:0:204:0: serial_number(ZA1AM36R0000R8318PN0) [ 34.967358] scsi 0:0:204:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.990094] mpt3sas_cm0: detecting: handle(0x00e5), sas_address(0x5000c500954ce0a2), phy(21) [ 34.998540] mpt3sas_cm0: REPORT_LUNS: handle(0x00e5), retries(0) [ 35.015805] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e5), lun(0) [ 35.022913] scsi 0:0:205:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.031574] scsi 0:0:205:0: SSP: handle(0x00e5), sas_addr(0x5000c500954ce0a2), phy(21), device_name(0x5000c500954ce0a0) [ 35.042341] scsi 0:0:205:0: enclosure logical id(0x50016360020accbd), slot(21) [ 35.049648] scsi 0:0:205:0: enclosure level(0x0000), connector name( C2 ) [ 35.056539] scsi 0:0:205:0: serial_number(ZA1ALKKW0000R8301RPL) [ 35.062460] scsi 0:0:205:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.085698] mpt3sas_cm0: detecting: handle(0x00e6), sas_address(0x5000c500954d3c06), phy(22) [ 35.094135] mpt3sas_cm0: REPORT_LUNS: handle(0x00e6), retries(0) [ 35.107017] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e6), lun(0) [ 35.114115] scsi 0:0:206:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.122781] scsi 0:0:206:0: SSP: handle(0x00e6), sas_addr(0x5000c500954d3c06), phy(22), device_name(0x5000c500954d3c04) [ 35.133554] scsi 0:0:206:0: enclosure logical id(0x50016360020accbd), slot(22) [ 35.140859] scsi 0:0:206:0: enclosure level(0x0000), connector name( C2 ) [ 35.147755] scsi 0:0:206:0: serial_number(ZA1ALTMD0000C8317U0L) [ 35.153682] scsi 0:0:206:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.177100] mpt3sas_cm0: detecting: handle(0x00e7), sas_address(0x5000c500954cd406), phy(23) [ 35.185542] mpt3sas_cm0: REPORT_LUNS: handle(0x00e7), retries(0) [ 35.197740] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e7), lun(0) [ 35.204844] scsi 0:0:207:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.213506] scsi 0:0:207:0: SSP: handle(0x00e7), sas_addr(0x5000c500954cd406), phy(23), device_name(0x5000c500954cd404) [ 35.224278] scsi 0:0:207:0: enclosure logical id(0x50016360020accbd), slot(23) [ 35.231585] scsi 0:0:207:0: enclosure level(0x0000), connector name( C2 ) [ 35.238476] scsi 0:0:207:0: serial_number(ZA1ALGD90000R8314QY2) [ 35.244396] scsi 0:0:207:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.267112] mpt3sas_cm0: detecting: handle(0x00e8), sas_address(0x5000c500954d2e36), phy(24) [ 35.275548] mpt3sas_cm0: REPORT_LUNS: handle(0x00e8), retries(0) [ 35.287410] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e8), lun(0) [ 35.294691] scsi 0:0:208:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.303344] scsi 0:0:208:0: SSP: handle(0x00e8), sas_addr(0x5000c500954d2e36), phy(24), device_name(0x5000c500954d2e34) [ 35.314118] scsi 0:0:208:0: enclosure logical id(0x50016360020accbd), slot(24) [ 35.321425] scsi 0:0:208:0: enclosure level(0x0000), connector name( C2 ) [ 35.328317] scsi 0:0:208:0: serial_number(ZA1ALV5Y0000R8314SNM) [ 35.334237] scsi 0:0:208:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.357103] mpt3sas_cm0: detecting: handle(0x00e9), sas_address(0x5000c500954cba3a), phy(25) [ 35.365547] mpt3sas_cm0: REPORT_LUNS: handle(0x00e9), retries(0) [ 35.380867] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e9), lun(0) [ 35.387996] scsi 0:0:209:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.396655] scsi 0:0:209:0: SSP: handle(0x00e9), sas_addr(0x5000c500954cba3a), phy(25), device_name(0x5000c500954cba38) [ 35.407426] scsi 0:0:209:0: enclosure logical id(0x50016360020accbd), slot(25) [ 35.414733] scsi 0:0:209:0: enclosure level(0x0000), connector name( C2 ) [ 35.421628] scsi 0:0:209:0: serial_number(ZA1ALLD60000R830XAFV) [ 35.427545] scsi 0:0:209:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.451101] mpt3sas_cm0: detecting: handle(0x00ea), sas_address(0x5000c500954e036e), phy(26) [ 35.459541] mpt3sas_cm0: REPORT_LUNS: handle(0x00ea), retries(0) [ 35.473482] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ea), lun(0) [ 35.480591] scsi 0:0:210:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.489256] scsi 0:0:210:0: SSP: handle(0x00ea), sas_addr(0x5000c500954e036e), phy(26), device_name(0x5000c500954e036c) [ 35.500032] scsi 0:0:210:0: enclosure logical id(0x50016360020accbd), slot(26) [ 35.507339] scsi 0:0:210:0: enclosure level(0x0000), connector name( C2 ) [ 35.514231] scsi 0:0:210:0: serial_number(ZA1AM1YQ0000R8317KKT) [ 35.520151] scsi 0:0:210:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.543111] mpt3sas_cm0: detecting: handle(0x00eb), sas_address(0x5000c500954d2aae), phy(27) [ 35.551556] mpt3sas_cm0: REPORT_LUNS: handle(0x00eb), retries(0) [ 35.569444] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00eb), lun(0) [ 35.576580] scsi 0:0:211:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.585242] scsi 0:0:211:0: SSP: handle(0x00eb), sas_addr(0x5000c500954d2aae), phy(27), device_name(0x5000c500954d2aac) [ 35.596017] scsi 0:0:211:0: enclosure logical id(0x50016360020accbd), slot(27) [ 35.603324] scsi 0:0:211:0: enclosure level(0x0000), connector name( C2 ) [ 35.610218] scsi 0:0:211:0: serial_number(ZA1ALV9V0000C8326DPQ) [ 35.616138] scsi 0:0:211:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.639112] mpt3sas_cm0: detecting: handle(0x00ec), sas_address(0x5000c500954cd14e), phy(28) [ 35.647549] mpt3sas_cm0: REPORT_LUNS: handle(0x00ec), retries(0) [ 35.668131] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ec), lun(0) [ 35.675242] scsi 0:0:212:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.683910] scsi 0:0:212:0: SSP: handle(0x00ec), sas_addr(0x5000c500954cd14e), phy(28), device_name(0x5000c500954cd14c) [ 35.694683] scsi 0:0:212:0: enclosure logical id(0x50016360020accbd), slot(28) [ 35.701989] scsi 0:0:212:0: enclosure level(0x0000), connector name( C2 ) [ 35.708882] scsi 0:0:212:0: serial_number(ZA1ALL1W0000R8301RZG) [ 35.714801] scsi 0:0:212:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.738111] mpt3sas_cm0: detecting: handle(0x00ed), sas_address(0x5000c50095389886), phy(29) [ 35.746551] mpt3sas_cm0: REPORT_LUNS: handle(0x00ed), retries(0) [ 35.764740] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ed), lun(0) [ 35.771861] scsi 0:0:213:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.780519] scsi 0:0:213:0: SSP: handle(0x00ed), sas_addr(0x5000c50095389886), phy(29), device_name(0x5000c50095389884) [ 35.791291] scsi 0:0:213:0: enclosure logical id(0x50016360020accbd), slot(29) [ 35.798597] scsi 0:0:213:0: enclosure level(0x0000), connector name( C2 ) [ 35.805492] scsi 0:0:213:0: serial_number(ZA1AD1B00000R829D6SU) [ 35.811411] scsi 0:0:213:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.834109] mpt3sas_cm0: detecting: handle(0x00ee), sas_address(0x5000c5009538d7fe), phy(30) [ 35.842546] mpt3sas_cm0: REPORT_LUNS: handle(0x00ee), retries(0) [ 35.863132] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ee), lun(0) [ 35.870258] scsi 0:0:214:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.878924] scsi 0:0:214:0: SSP: handle(0x00ee), sas_addr(0x5000c5009538d7fe), phy(30), device_name(0x5000c5009538d7fc) [ 35.889695] scsi 0:0:214:0: enclosure logical id(0x50016360020accbd), slot(30) [ 35.897001] scsi 0:0:214:0: enclosure level(0x0000), connector name( C2 ) [ 35.903896] scsi 0:0:214:0: serial_number(ZA1ACZYX0000R82898PK) [ 35.909815] scsi 0:0:214:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.933120] mpt3sas_cm0: detecting: handle(0x00ef), sas_address(0x5000c500954d3576), phy(31) [ 35.941556] mpt3sas_cm0: REPORT_LUNS: handle(0x00ef), retries(0) [ 35.952798] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ef), lun(0) [ 35.959917] scsi 0:0:215:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.968577] scsi 0:0:215:0: SSP: handle(0x00ef), sas_addr(0x5000c500954d3576), phy(31), device_name(0x5000c500954d3574) [ 35.979347] scsi 0:0:215:0: enclosure logical id(0x50016360020accbd), slot(31) [ 35.986652] scsi 0:0:215:0: enclosure level(0x0000), connector name( C2 ) [ 35.993545] scsi 0:0:215:0: serial_number(ZA1ALY010000R83027E2) [ 35.999464] scsi 0:0:215:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.022121] mpt3sas_cm0: detecting: handle(0x00f0), sas_address(0x5000c5009537d822), phy(32) [ 36.030556] mpt3sas_cm0: REPORT_LUNS: handle(0x00f0), retries(0) [ 36.046102] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f0), lun(0) [ 36.053222] scsi 0:0:216:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.061888] scsi 0:0:216:0: SSP: handle(0x00f0), sas_addr(0x5000c5009537d822), phy(32), device_name(0x5000c5009537d820) [ 36.072663] scsi 0:0:216:0: enclosure logical id(0x50016360020accbd), slot(32) [ 36.079970] scsi 0:0:216:0: enclosure level(0x0000), connector name( C2 ) [ 36.086864] scsi 0:0:216:0: serial_number(ZA1ACKTS0000R829CSFQ) [ 36.092783] scsi 0:0:216:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.116123] mpt3sas_cm0: detecting: handle(0x00f1), sas_address(0x5000c5009538501e), phy(33) [ 36.124559] mpt3sas_cm0: REPORT_LUNS: handle(0x00f1), retries(0) [ 36.139678] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f1), lun(0) [ 36.146776] scsi 0:0:217:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.155432] scsi 0:0:217:0: SSP: handle(0x00f1), sas_addr(0x5000c5009538501e), phy(33), device_name(0x5000c5009538501c) [ 36.166205] scsi 0:0:217:0: enclosure logical id(0x50016360020accbd), slot(33) [ 36.173510] scsi 0:0:217:0: enclosure level(0x0000), connector name( C2 ) [ 36.180404] scsi 0:0:217:0: serial_number(ZA1ACX560000R829D7UM) [ 36.186324] scsi 0:0:217:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.209134] mpt3sas_cm0: detecting: handle(0x00f2), sas_address(0x5000c5009538cb6a), phy(34) [ 36.217572] mpt3sas_cm0: REPORT_LUNS: handle(0x00f2), retries(0) [ 36.224123] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f2), lun(0) [ 36.231223] scsi 0:0:218:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.239891] scsi 0:0:218:0: SSP: handle(0x00f2), sas_addr(0x5000c5009538cb6a), phy(34), device_name(0x5000c5009538cb68) [ 36.250665] scsi 0:0:218:0: enclosure logical id(0x50016360020accbd), slot(34) [ 36.257969] scsi 0:0:218:0: enclosure level(0x0000), connector name( C2 ) [ 36.264864] scsi 0:0:218:0: serial_number(ZA1AD0830000R829CTL8) [ 36.270783] scsi 0:0:218:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.294125] mpt3sas_cm0: detecting: handle(0x00f3), sas_address(0x5000c500954d2be6), phy(35) [ 36.302569] mpt3sas_cm0: REPORT_LUNS: handle(0x00f3), retries(0) [ 36.315701] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f3), lun(0) [ 36.322821] scsi 0:0:219:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.331485] scsi 0:0:219:0: SSP: handle(0x00f3), sas_addr(0x5000c500954d2be6), phy(35), device_name(0x5000c500954d2be4) [ 36.342256] scsi 0:0:219:0: enclosure logical id(0x50016360020accbd), slot(35) [ 36.349562] scsi 0:0:219:0: enclosure level(0x0000), connector name( C2 ) [ 36.356453] scsi 0:0:219:0: serial_number(ZA1ALYAL0000R8314T6V) [ 36.362375] scsi 0:0:219:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.385125] mpt3sas_cm0: detecting: handle(0x00f4), sas_address(0x5000c50095387dd2), phy(36) [ 36.393562] mpt3sas_cm0: REPORT_LUNS: handle(0x00f4), retries(0) [ 36.400117] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f4), lun(0) [ 36.407230] scsi 0:0:220:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.415893] scsi 0:0:220:0: SSP: handle(0x00f4), sas_addr(0x5000c50095387dd2), phy(36), device_name(0x5000c50095387dd0) [ 36.426664] scsi 0:0:220:0: enclosure logical id(0x50016360020accbd), slot(36) [ 36.433970] scsi 0:0:220:0: enclosure level(0x0000), connector name( C2 ) [ 36.440863] scsi 0:0:220:0: serial_number(ZA1ACWHH0000R8289ZWP) [ 36.446782] scsi 0:0:220:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.470134] mpt3sas_cm0: detecting: handle(0x00f5), sas_address(0x5000c500953896a2), phy(37) [ 36.478579] mpt3sas_cm0: REPORT_LUNS: handle(0x00f5), retries(0) [ 36.491690] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f5), lun(0) [ 36.498801] scsi 0:0:221:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.507470] scsi 0:0:221:0: SSP: handle(0x00f5), sas_addr(0x5000c500953896a2), phy(37), device_name(0x5000c500953896a0) [ 36.518245] scsi 0:0:221:0: enclosure logical id(0x50016360020accbd), slot(37) [ 36.525552] scsi 0:0:221:0: enclosure level(0x0000), connector name( C2 ) [ 36.532444] scsi 0:0:221:0: serial_number(ZA1AD10R0000R8289ZQ3) [ 36.538363] scsi 0:0:221:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.564142] mpt3sas_cm0: detecting: handle(0x00f6), sas_address(0x5000c5009537beb2), phy(38) [ 36.572586] mpt3sas_cm0: REPORT_LUNS: handle(0x00f6), retries(0) [ 36.588200] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f6), lun(0) [ 36.595317] scsi 0:0:222:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.603984] scsi 0:0:222:0: SSP: handle(0x00f6), sas_addr(0x5000c5009537beb2), phy(38), device_name(0x5000c5009537beb0) [ 36.614760] scsi 0:0:222:0: enclosure logical id(0x50016360020accbd), slot(38) [ 36.622066] scsi 0:0:222:0: enclosure level(0x0000), connector name( C2 ) [ 36.628961] scsi 0:0:222:0: serial_number(ZA1ACL4N0000R82896AH) [ 36.634880] scsi 0:0:222:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.658135] mpt3sas_cm0: detecting: handle(0x00f7), sas_address(0x5000c500954dcc5e), phy(39) [ 36.666578] mpt3sas_cm0: REPORT_LUNS: handle(0x00f7), retries(0) [ 36.682933] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f7), lun(0) [ 36.690069] scsi 0:0:223:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.698729] scsi 0:0:223:0: SSP: handle(0x00f7), sas_addr(0x5000c500954dcc5e), phy(39), device_name(0x5000c500954dcc5c) [ 36.709497] scsi 0:0:223:0: enclosure logical id(0x50016360020accbd), slot(39) [ 36.716803] scsi 0:0:223:0: enclosure level(0x0000), connector name( C2 ) [ 36.723699] scsi 0:0:223:0: serial_number(ZA1AM66D0000R8313JDM) [ 36.729626] scsi 0:0:223:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.753138] mpt3sas_cm0: detecting: handle(0x00f8), sas_address(0x5000c500954e02c6), phy(40) [ 36.761576] mpt3sas_cm0: REPORT_LUNS: handle(0x00f8), retries(0) [ 36.778563] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f8), lun(0) [ 36.785695] scsi 0:0:224:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.794361] scsi 0:0:224:0: SSP: handle(0x00f8), sas_addr(0x5000c500954e02c6), phy(40), device_name(0x5000c500954e02c4) [ 36.805137] scsi 0:0:224:0: enclosure logical id(0x50016360020accbd), slot(40) [ 36.812444] scsi 0:0:224:0: enclosure level(0x0000), connector name( C2 ) [ 36.819336] scsi 0:0:224:0: serial_number(ZA1AM68A0000R8317LWC) [ 36.825254] scsi 0:0:224:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.848146] mpt3sas_cm0: detecting: handle(0x00f9), sas_address(0x5000c500954dd88e), phy(41) [ 36.856582] mpt3sas_cm0: REPORT_LUNS: handle(0x00f9), retries(0) [ 36.875509] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f9), lun(0) [ 36.882633] scsi 0:0:225:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.891305] scsi 0:0:225:0: SSP: handle(0x00f9), sas_addr(0x5000c500954dd88e), phy(41), device_name(0x5000c500954dd88c) [ 36.902076] scsi 0:0:225:0: enclosure logical id(0x50016360020accbd), slot(41) [ 36.909381] scsi 0:0:225:0: enclosure level(0x0000), connector name( C2 ) [ 36.916275] scsi 0:0:225:0: serial_number(ZA1AM2JK0000R831459F) [ 36.922195] scsi 0:0:225:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.945144] mpt3sas_cm0: detecting: handle(0x00fa), sas_address(0x5000c500954d3bb6), phy(42) [ 36.953583] mpt3sas_cm0: REPORT_LUNS: handle(0x00fa), retries(0) [ 36.973208] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fa), lun(0) [ 36.980317] scsi 0:0:226:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.988985] scsi 0:0:226:0: SSP: handle(0x00fa), sas_addr(0x5000c500954d3bb6), phy(42), device_name(0x5000c500954d3bb4) [ 36.999762] scsi 0:0:226:0: enclosure logical id(0x50016360020accbd), slot(42) [ 37.007068] scsi 0:0:226:0: enclosure level(0x0000), connector name( C2 ) [ 37.013960] scsi 0:0:226:0: serial_number(ZA1ALTMW0000R8314UZP) [ 37.019881] scsi 0:0:226:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.043142] mpt3sas_cm0: detecting: handle(0x00fb), sas_address(0x5000c5009538239e), phy(43) [ 37.051577] mpt3sas_cm0: REPORT_LUNS: handle(0x00fb), retries(0) [ 37.065890] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fb), lun(0) [ 37.073005] scsi 0:0:227:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.081666] scsi 0:0:227:0: SSP: handle(0x00fb), sas_addr(0x5000c5009538239e), phy(43), device_name(0x5000c5009538239c) [ 37.092437] scsi 0:0:227:0: enclosure logical id(0x50016360020accbd), slot(43) [ 37.099741] scsi 0:0:227:0: enclosure level(0x0000), connector name( C2 ) [ 37.106635] scsi 0:0:227:0: serial_number(ZA1ACMYS0000R829D5HG) [ 37.112556] scsi 0:0:227:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.134751] mpt3sas_cm0: detecting: handle(0x00fc), sas_address(0x5000c500954d07a6), phy(44) [ 37.143189] mpt3sas_cm0: REPORT_LUNS: handle(0x00fc), retries(0) [ 37.156337] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fc), lun(0) [ 37.163447] scsi 0:0:228:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.172101] scsi 0:0:228:0: SSP: handle(0x00fc), sas_addr(0x5000c500954d07a6), phy(44), device_name(0x5000c500954d07a4) [ 37.182876] scsi 0:0:228:0: enclosure logical id(0x50016360020accbd), slot(44) [ 37.190181] scsi 0:0:228:0: enclosure level(0x0000), connector name( C2 ) [ 37.197075] scsi 0:0:228:0: serial_number(ZA1ALF450000R8314TJR) [ 37.202992] scsi 0:0:228:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.226147] mpt3sas_cm0: detecting: handle(0x00fd), sas_address(0x5000c5009521d99a), phy(45) [ 37.234588] mpt3sas_cm0: REPORT_LUNS: handle(0x00fd), retries(0) [ 37.248875] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fd), lun(0) [ 37.256000] scsi 0:0:229:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.264661] scsi 0:0:229:0: SSP: handle(0x00fd), sas_addr(0x5000c5009521d99a), phy(45), device_name(0x5000c5009521d998) [ 37.275438] scsi 0:0:229:0: enclosure logical id(0x50016360020accbd), slot(45) [ 37.282743] scsi 0:0:229:0: enclosure level(0x0000), connector name( C2 ) [ 37.289637] scsi 0:0:229:0: serial_number(ZA1A5SLF0000R824DGWD) [ 37.295557] scsi 0:0:229:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.318150] mpt3sas_cm0: detecting: handle(0x00fe), sas_address(0x5000c50095372306), phy(46) [ 37.326586] mpt3sas_cm0: REPORT_LUNS: handle(0x00fe), retries(0) [ 37.342894] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fe), lun(0) [ 37.350016] scsi 0:0:230:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.358674] scsi 0:0:230:0: SSP: handle(0x00fe), sas_addr(0x5000c50095372306), phy(46), device_name(0x5000c50095372304) [ 37.369448] scsi 0:0:230:0: enclosure logical id(0x50016360020accbd), slot(46) [ 37.376754] scsi 0:0:230:0: enclosure level(0x0000), connector name( C2 ) [ 37.383647] scsi 0:0:230:0: serial_number(ZA1A548E0000R824DE6T) [ 37.389568] scsi 0:0:230:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.413161] mpt3sas_cm0: detecting: handle(0x00ff), sas_address(0x5000c50095382566), phy(47) [ 37.421604] mpt3sas_cm0: REPORT_LUNS: handle(0x00ff), retries(0) [ 37.441383] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ff), lun(0) [ 37.448518] scsi 0:0:231:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.457179] scsi 0:0:231:0: SSP: handle(0x00ff), sas_addr(0x5000c50095382566), phy(47), device_name(0x5000c50095382564) [ 37.467954] scsi 0:0:231:0: enclosure logical id(0x50016360020accbd), slot(47) [ 37.475260] scsi 0:0:231:0: enclosure level(0x0000), connector name( C2 ) [ 37.482155] scsi 0:0:231:0: serial_number(ZA1ACN0Z0000R829SWAY) [ 37.488073] scsi 0:0:231:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.511162] mpt3sas_cm0: detecting: handle(0x0100), sas_address(0x5000c5009538c656), phy(48) [ 37.519598] mpt3sas_cm0: REPORT_LUNS: handle(0x0100), retries(0) [ 37.531142] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0100), lun(0) [ 37.538262] scsi 0:0:232:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.546930] scsi 0:0:232:0: SSP: handle(0x0100), sas_addr(0x5000c5009538c656), phy(48), device_name(0x5000c5009538c654) [ 37.557702] scsi 0:0:232:0: enclosure logical id(0x50016360020accbd), slot(48) [ 37.565007] scsi 0:0:232:0: enclosure level(0x0000), connector name( C2 ) [ 37.571910] scsi 0:0:232:0: serial_number(ZA1AD0CS0000R829CFRR) [ 37.577832] scsi 0:0:232:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.601160] mpt3sas_cm0: detecting: handle(0x0101), sas_address(0x5000c500953801c6), phy(49) [ 37.609597] mpt3sas_cm0: REPORT_LUNS: handle(0x0101), retries(0) [ 37.622517] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0101), lun(0) [ 37.629664] scsi 0:0:233:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.638323] scsi 0:0:233:0: SSP: handle(0x0101), sas_addr(0x5000c500953801c6), phy(49), device_name(0x5000c500953801c4) [ 37.649093] scsi 0:0:233:0: enclosure logical id(0x50016360020accbd), slot(49) [ 37.656400] scsi 0:0:233:0: enclosure level(0x0000), connector name( C2 ) [ 37.663293] scsi 0:0:233:0: serial_number(ZA1ACJB90000R829D764) [ 37.669211] scsi 0:0:233:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.692172] mpt3sas_cm0: detecting: handle(0x0102), sas_address(0x5000c50095383876), phy(50) [ 37.700617] mpt3sas_cm0: REPORT_LUNS: handle(0x0102), retries(0) [ 37.716498] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0102), lun(0) [ 37.723624] scsi 0:0:234:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.732280] scsi 0:0:234:0: SSP: handle(0x0102), sas_addr(0x5000c50095383876), phy(50), device_name(0x5000c50095383874) [ 37.743051] scsi 0:0:234:0: enclosure logical id(0x50016360020accbd), slot(50) [ 37.750358] scsi 0:0:234:0: enclosure level(0x0000), connector name( C2 ) [ 37.757249] scsi 0:0:234:0: serial_number(ZA1ACHHJ0000R8289ZFE) [ 37.763170] scsi 0:0:234:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.786161] mpt3sas_cm0: detecting: handle(0x0103), sas_address(0x5000c50095380fc6), phy(51) [ 37.794599] mpt3sas_cm0: REPORT_LUNS: handle(0x0103), retries(0) [ 37.814596] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0103), lun(0) [ 37.821747] scsi 0:0:235:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.830404] scsi 0:0:235:0: SSP: handle(0x0103), sas_addr(0x5000c50095380fc6), phy(51), device_name(0x5000c50095380fc4) [ 37.841178] scsi 0:0:235:0: enclosure logical id(0x50016360020accbd), slot(51) [ 37.848483] scsi 0:0:235:0: enclosure level(0x0000), connector name( C2 ) [ 37.855390] scsi 0:0:235:0: serial_number(ZA1ACNAB0000R829CXJ8) [ 37.861314] scsi 0:0:235:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.884169] mpt3sas_cm0: detecting: handle(0x0104), sas_address(0x5000c5009538819a), phy(52) [ 37.892606] mpt3sas_cm0: REPORT_LUNS: handle(0x0104), retries(0) [ 37.910122] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0104), lun(0) [ 37.917232] scsi 0:0:236:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.925886] scsi 0:0:236:0: SSP: handle(0x0104), sas_addr(0x5000c5009538819a), phy(52), device_name(0x5000c50095388198) [ 37.936661] scsi 0:0:236:0: enclosure logical id(0x50016360020accbd), slot(52) [ 37.943967] scsi 0:0:236:0: enclosure level(0x0000), connector name( C2 ) [ 37.950862] scsi 0:0:236:0: serial_number(ZA1AD1NN0000R829CXLQ) [ 37.956780] scsi 0:0:236:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.980173] mpt3sas_cm0: detecting: handle(0x0105), sas_address(0x5000c500954d99fe), phy(53) [ 37.988609] mpt3sas_cm0: REPORT_LUNS: handle(0x0105), retries(0) [ 38.006371] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0105), lun(0) [ 38.013501] scsi 0:0:237:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.022158] scsi 0:0:237:0: SSP: handle(0x0105), sas_addr(0x5000c500954d99fe), phy(53), device_name(0x5000c500954d99fc) [ 38.032934] scsi 0:0:237:0: enclosure logical id(0x50016360020accbd), slot(53) [ 38.040238] scsi 0:0:237:0: enclosure level(0x0000), connector name( C2 ) [ 38.047131] scsi 0:0:237:0: serial_number(ZA1AM3NP0000R830285F) [ 38.053052] scsi 0:0:237:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.076177] mpt3sas_cm0: detecting: handle(0x0106), sas_address(0x5000c50095381d46), phy(54) [ 38.084623] mpt3sas_cm0: REPORT_LUNS: handle(0x0106), retries(0) [ 38.104818] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0106), lun(0) [ 38.111941] scsi 0:0:238:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.120596] scsi 0:0:238:0: SSP: handle(0x0106), sas_addr(0x5000c50095381d46), phy(54), device_name(0x5000c50095381d44) [ 38.131372] scsi 0:0:238:0: enclosure logical id(0x50016360020accbd), slot(54) [ 38.138678] scsi 0:0:238:0: enclosure level(0x0000), connector name( C2 ) [ 38.145570] scsi 0:0:238:0: serial_number(ZA1ACHRE0000R829CSK2) [ 38.151489] scsi 0:0:238:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.177167] mpt3sas_cm0: detecting: handle(0x0107), sas_address(0x5000c500954d2b72), phy(55) [ 38.185621] mpt3sas_cm0: REPORT_LUNS: handle(0x0107), retries(0) [ 38.198173] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0107), lun(0) [ 38.205285] scsi 0:0:239:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.213942] scsi 0:0:239:0: SSP: handle(0x0107), sas_addr(0x5000c500954d2b72), phy(55), device_name(0x5000c500954d2b70) [ 38.224713] scsi 0:0:239:0: enclosure logical id(0x50016360020accbd), slot(55) [ 38.232021] scsi 0:0:239:0: enclosure level(0x0000), connector name( C2 ) [ 38.238916] scsi 0:0:239:0: serial_number(ZA1ALV9B0000C8323VXP) [ 38.244843] scsi 0:0:239:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.268178] mpt3sas_cm0: detecting: handle(0x0108), sas_address(0x5000c500954ccdea), phy(56) [ 38.276617] mpt3sas_cm0: REPORT_LUNS: handle(0x0108), retries(0) [ 38.296248] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0108), lun(0) [ 38.303364] scsi 0:0:240:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.312027] scsi 0:0:240:0: SSP: handle(0x0108), sas_addr(0x5000c500954ccdea), phy(56), device_name(0x5000c500954ccde8) [ 38.322796] scsi 0:0:240:0: enclosure logical id(0x50016360020accbd), slot(56) [ 38.330104] scsi 0:0:240:0: enclosure level(0x0000), connector name( C2 ) [ 38.336997] scsi 0:0:240:0: serial_number(ZA1ALGLA0000R8314QAF) [ 38.342915] scsi 0:0:240:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.397187] mpt3sas_cm0: detecting: handle(0x0109), sas_address(0x5000c5009537e98a), phy(57) [ 38.405626] mpt3sas_cm0: REPORT_LUNS: handle(0x0109), retries(0) [ 38.418303] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0109), lun(0) [ 38.425414] scsi 0:0:241:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.434072] scsi 0:0:241:0: SSP: handle(0x0109), sas_addr(0x5000c5009537e98a), phy(57), device_name(0x5000c5009537e988) [ 38.444846] scsi 0:0:241:0: enclosure logical id(0x50016360020accbd), slot(57) [ 38.452150] scsi 0:0:241:0: enclosure level(0x0000), connector name( C2 ) [ 38.459045] scsi 0:0:241:0: serial_number(ZA1ACK9D0000R829D59Z) [ 38.464965] scsi 0:0:241:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.488181] mpt3sas_cm0: detecting: handle(0x010a), sas_address(0x5000c5009538819e), phy(58) [ 38.496618] mpt3sas_cm0: REPORT_LUNS: handle(0x010a), retries(0) [ 38.515286] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010a), lun(0) [ 38.522422] scsi 0:0:242:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.531080] scsi 0:0:242:0: SSP: handle(0x010a), sas_addr(0x5000c5009538819e), phy(58), device_name(0x5000c5009538819c) [ 38.541852] scsi 0:0:242:0: enclosure logical id(0x50016360020accbd), slot(58) [ 38.549158] scsi 0:0:242:0: enclosure level(0x0000), connector name( C2 ) [ 38.556058] scsi 0:0:242:0: serial_number(ZA1AD1NX0000R829CS3G) [ 38.561983] scsi 0:0:242:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.585185] mpt3sas_cm0: detecting: handle(0x010b), sas_address(0x5000c500954d962a), phy(59) [ 38.593628] mpt3sas_cm0: REPORT_LUNS: handle(0x010b), retries(0) [ 38.615172] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010b), lun(0) [ 38.622297] scsi 0:0:243:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.630959] scsi 0:0:243:0: SSP: handle(0x010b), sas_addr(0x5000c500954d962a), phy(59), device_name(0x5000c500954d9628) [ 38.641731] scsi 0:0:243:0: enclosure logical id(0x50016360020accbd), slot(59) [ 38.649037] scsi 0:0:243:0: enclosure level(0x0000), connector name( C2 ) [ 38.655931] scsi 0:0:243:0: serial_number(ZA1AM7DR0000R8317MX0) [ 38.661849] scsi 0:0:243:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.689383] mpt3sas_cm0: detecting: handle(0x010d), sas_address(0x50016360020accbd), phy(76) [ 38.697837] mpt3sas_cm0: REPORT_LUNS: handle(0x010d), retries(0) [ 38.704039] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010d), lun(0) [ 38.710525] scsi 0:0:244:0: Enclosure QCT JB4602 SIM 1 1500 PQ: 0 ANSI: 6 [ 38.718996] scsi 0:0:244:0: set ignore_delay_remove for handle(0x010d) [ 38.725522] scsi 0:0:244:0: SES: handle(0x010d), sas_addr(0x50016360020accbd), phy(76), device_name(0x50016360020accbd) [ 38.736295] scsi 0:0:244:0: enclosure logical id(0x50016360020accbd), slot(60) [ 38.743600] scsi 0:0:244:0: enclosure level(0x0000), connector name( C2 ) [ 38.750495] scsi 0:0:244:0: serial_number(EO0161900025 ) [ 38.756067] scsi 0:0:244:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.782731] mpt3sas_cm0: port enable: SUCCESS [ 39.283474] sd 0:0:15:0: [sdo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.288913] sd 0:0:58:0: [sdbf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.288922] sd 0:0:58:0: [sdbf] 4096-byte physical blocks [ 39.289370] sd 0:0:76:0: [sdbw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.289372] sd 0:0:76:0: [sdbw] 4096-byte physical blocks [ 39.289780] sd 0:0:2:0: [sdb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.289782] sd 0:0:2:0: [sdb] 4096-byte physical blocks [ 39.290018] sd 0:0:100:0: [sdcu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290020] sd 0:0:100:0: [sdcu] 4096-byte physical blocks [ 39.290053] sd 0:0:101:0: [sdcv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290056] sd 0:0:101:0: [sdcv] 4096-byte physical blocks [ 39.290265] sd 0:0:109:0: [sddd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290269] sd 0:0:109:0: [sddd] 4096-byte physical blocks [ 39.290544] sd 0:0:116:0: [sddk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290546] sd 0:0:116:0: [sddk] 4096-byte physical blocks [ 39.290602] sd 0:0:117:0: [sddl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290603] sd 0:0:117:0: [sddl] 4096-byte physical blocks [ 39.290765] sd 0:0:123:0: [sddq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290767] sd 0:0:123:0: [sddq] 4096-byte physical blocks [ 39.290769] sd 0:0:3:0: [sdc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290771] sd 0:0:3:0: [sdc] 4096-byte physical blocks [ 39.290915] sd 0:0:126:0: [sddt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290916] sd 0:0:126:0: [sddt] 4096-byte physical blocks [ 39.290998] sd 0:0:128:0: [sddv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.290999] sd 0:0:128:0: [sddv] 4096-byte physical blocks [ 39.291095] sd 0:0:130:0: [sddx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291096] sd 0:0:130:0: [sddx] 4096-byte physical blocks [ 39.291112] sd 0:0:127:0: [sddu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291113] sd 0:0:127:0: [sddu] 4096-byte physical blocks [ 39.291166] sd 0:0:132:0: [sddz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291167] sd 0:0:132:0: [sddz] 4096-byte physical blocks [ 39.291221] sd 0:0:137:0: [sdee] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291222] sd 0:0:137:0: [sdee] 4096-byte physical blocks [ 39.291267] sd 0:0:58:0: [sdbf] Write Protect is off [ 39.291268] sd 0:0:58:0: [sdbf] Mode Sense: db 00 10 08 [ 39.291274] sd 0:0:134:0: [sdeb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291284] sd 0:0:134:0: [sdeb] 4096-byte physical blocks [ 39.291393] sd 0:0:138:0: [sdef] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291394] sd 0:0:138:0: [sdef] 4096-byte physical blocks [ 39.291416] sd 0:0:124:0: [sddr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291417] sd 0:0:124:0: [sddr] 4096-byte physical blocks [ 39.291499] sd 0:0:141:0: [sdei] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291501] sd 0:0:141:0: [sdei] 4096-byte physical blocks [ 39.291565] sd 0:0:125:0: [sdds] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291571] sd 0:0:125:0: [sdds] 4096-byte physical blocks [ 39.291617] sd 0:0:143:0: [sdek] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291618] sd 0:0:143:0: [sdek] 4096-byte physical blocks [ 39.291627] sd 0:0:76:0: [sdbw] Write Protect is off [ 39.291629] sd 0:0:76:0: [sdbw] Mode Sense: db 00 10 08 [ 39.291658] sd 0:0:46:0: [sdat] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291660] sd 0:0:46:0: [sdat] 4096-byte physical blocks [ 39.291716] sd 0:0:54:0: [sdbb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291717] sd 0:0:54:0: [sdbb] 4096-byte physical blocks [ 39.291777] sd 0:0:146:0: [sden] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291778] sd 0:0:146:0: [sden] 4096-byte physical blocks [ 39.291788] sd 0:0:133:0: [sdea] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291789] sd 0:0:133:0: [sdea] 4096-byte physical blocks [ 39.291807] sd 0:0:42:0: [sdap] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291808] sd 0:0:42:0: [sdap] 4096-byte physical blocks [ 39.291821] sd 0:0:145:0: [sdem] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291823] sd 0:0:145:0: [sdem] 4096-byte physical blocks [ 39.291840] sd 0:0:40:0: [sdan] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291842] sd 0:0:40:0: [sdan] 4096-byte physical blocks [ 39.291899] sd 0:0:10:0: [sdj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291901] sd 0:0:10:0: [sdj] 4096-byte physical blocks [ 39.291929] sd 0:0:1:0: [sda] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291931] sd 0:0:43:0: [sdaq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291932] sd 0:0:1:0: [sda] 4096-byte physical blocks [ 39.291934] sd 0:0:43:0: [sdaq] 4096-byte physical blocks [ 39.291987] sd 0:0:21:0: [sdu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.291989] sd 0:0:21:0: [sdu] 4096-byte physical blocks [ 39.292055] sd 0:0:29:0: [sdac] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292056] sd 0:0:29:0: [sdac] 4096-byte physical blocks [ 39.292068] sd 0:0:12:0: [sdl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292070] sd 0:0:12:0: [sdl] 4096-byte physical blocks [ 39.292080] sd 0:0:4:0: [sdd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292082] sd 0:0:4:0: [sdd] 4096-byte physical blocks [ 39.292097] sd 0:0:28:0: [sdab] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292098] sd 0:0:28:0: [sdab] 4096-byte physical blocks [ 39.292127] sd 0:0:30:0: [sdad] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292128] sd 0:0:30:0: [sdad] 4096-byte physical blocks [ 39.292143] sd 0:0:6:0: [sdf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292145] sd 0:0:6:0: [sdf] 4096-byte physical blocks [ 39.292157] sd 0:0:24:0: [sdx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292158] sd 0:0:24:0: [sdx] 4096-byte physical blocks [ 39.292182] sd 0:0:16:0: [sdp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292183] sd 0:0:16:0: [sdp] 4096-byte physical blocks [ 39.292206] sd 0:0:23:0: [sdw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292208] sd 0:0:23:0: [sdw] 4096-byte physical blocks [ 39.292210] sd 0:0:129:0: [sddw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292212] sd 0:0:129:0: [sddw] 4096-byte physical blocks [ 39.292231] sd 0:0:139:0: [sdeg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292234] sd 0:0:139:0: [sdeg] 4096-byte physical blocks [ 39.292248] sd 0:0:8:0: [sdh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292250] sd 0:0:8:0: [sdh] 4096-byte physical blocks [ 39.292270] sd 0:0:5:0: [sde] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292272] sd 0:0:5:0: [sde] 4096-byte physical blocks [ 39.292277] sd 0:0:100:0: [sdcu] Write Protect is off [ 39.292278] sd 0:0:100:0: [sdcu] Mode Sense: db 00 10 08 [ 39.292281] sd 0:0:19:0: [sds] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292283] sd 0:0:19:0: [sds] 4096-byte physical blocks [ 39.292326] sd 0:0:101:0: [sdcv] Write Protect is off [ 39.292327] sd 0:0:101:0: [sdcv] Mode Sense: db 00 10 08 [ 39.292433] sd 0:0:150:0: [sder] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292434] sd 0:0:150:0: [sder] 4096-byte physical blocks [ 39.292486] sd 0:0:136:0: [sded] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292488] sd 0:0:136:0: [sded] 4096-byte physical blocks [ 39.292532] sd 0:0:58:0: [sdbf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.292538] sd 0:0:109:0: [sddd] Write Protect is off [ 39.292540] sd 0:0:109:0: [sddd] Mode Sense: db 00 10 08 [ 39.292558] sd 0:0:151:0: [sdes] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292559] sd 0:0:151:0: [sdes] 4096-byte physical blocks [ 39.292780] sd 0:0:2:0: [sdb] Write Protect is off [ 39.292782] sd 0:0:2:0: [sdb] Mode Sense: db 00 10 08 [ 39.292826] sd 0:0:116:0: [sddk] Write Protect is off [ 39.292827] sd 0:0:116:0: [sddk] Mode Sense: db 00 10 08 [ 39.292860] sd 0:0:117:0: [sddl] Write Protect is off [ 39.292861] sd 0:0:117:0: [sddl] Mode Sense: db 00 10 08 [ 39.292918] sd 0:0:152:0: [sdet] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.292919] sd 0:0:152:0: [sdet] 4096-byte physical blocks [ 39.292956] sd 0:0:76:0: [sdbw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.293056] sd 0:0:147:0: [sdeo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293057] sd 0:0:147:0: [sdeo] 4096-byte physical blocks [ 39.293423] sd 0:0:164:0: [sdff] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293424] sd 0:0:164:0: [sdff] 4096-byte physical blocks [ 39.293433] sd 0:0:162:0: [sdfd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293434] sd 0:0:162:0: [sdfd] 4096-byte physical blocks [ 39.293507] sd 0:0:77:0: [sdbx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293508] sd 0:0:77:0: [sdbx] 4096-byte physical blocks [ 39.293523] sd 0:0:137:0: [sdee] Write Protect is off [ 39.293524] sd 0:0:137:0: [sdee] Mode Sense: db 00 10 08 [ 39.293582] sd 0:0:165:0: [sdfg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293584] sd 0:0:165:0: [sdfg] 4096-byte physical blocks [ 39.293601] sd 0:0:100:0: [sdcu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.293653] sd 0:0:101:0: [sdcv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.293659] sd 0:0:140:0: [sdeh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293660] sd 0:0:140:0: [sdeh] 4096-byte physical blocks [ 39.293663] sd 0:0:73:0: [sdbt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293664] sd 0:0:73:0: [sdbt] 4096-byte physical blocks [ 39.293668] sd 0:0:168:0: [sdfj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293669] sd 0:0:168:0: [sdfj] 4096-byte physical blocks [ 39.293815] sd 0:0:109:0: [sddd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.293819] sd 0:0:48:0: [sdav] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.293821] sd 0:0:48:0: [sdav] 4096-byte physical blocks [ 39.293934] sd 0:0:123:0: [sddq] Write Protect is off [ 39.293935] sd 0:0:123:0: [sddq] Mode Sense: db 00 10 08 [ 39.294010] sd 0:0:133:0: [sdea] Write Protect is off [ 39.294011] sd 0:0:133:0: [sdea] Mode Sense: db 00 10 08 [ 39.294012] sd 0:0:135:0: [sdec] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294017] sd 0:0:135:0: [sdec] 4096-byte physical blocks [ 39.294025] sd 0:0:96:0: [sdcq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294027] sd 0:0:96:0: [sdcq] 4096-byte physical blocks [ 39.294048] sd 0:0:54:0: [sdbb] Write Protect is off [ 39.294049] sd 0:0:54:0: [sdbb] Mode Sense: db 00 10 08 [ 39.294051] sd 0:0:49:0: [sdaw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294052] sd 0:0:49:0: [sdaw] 4096-byte physical blocks [ 39.294092] sd 0:0:126:0: [sddt] Write Protect is off [ 39.294093] sd 0:0:126:0: [sddt] Mode Sense: db 00 10 08 [ 39.294102] sd 0:0:116:0: [sddk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.294129] sd 0:0:117:0: [sddl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.294157] sd 0:0:170:0: [sdfl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294158] sd 0:0:170:0: [sdfl] 4096-byte physical blocks [ 39.294184] sd 0:0:180:0: [sdfv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294185] sd 0:0:180:0: [sdfv] 4096-byte physical blocks [ 39.294218] sd 0:0:128:0: [sddv] Write Protect is off [ 39.294220] sd 0:0:128:0: [sddv] Mode Sense: db 00 10 08 [ 39.294286] sd 0:0:45:0: [sdas] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294287] sd 0:0:45:0: [sdas] 4096-byte physical blocks [ 39.294288] sd 0:0:130:0: [sddx] Write Protect is off [ 39.294290] sd 0:0:130:0: [sddx] Mode Sense: db 00 10 08 [ 39.294299] sd 0:0:127:0: [sddu] Write Protect is off [ 39.294300] sd 0:0:127:0: [sddu] Mode Sense: db 00 10 08 [ 39.294310] sd 0:0:3:0: [sdc] Write Protect is off [ 39.294312] sd 0:0:3:0: [sdc] Mode Sense: db 00 10 08 [ 39.294358] sd 0:0:46:0: [sdat] Write Protect is off [ 39.294360] sd 0:0:46:0: [sdat] Mode Sense: db 00 10 08 [ 39.294394] sd 0:0:171:0: [sdfm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294395] sd 0:0:171:0: [sdfm] 4096-byte physical blocks [ 39.294413] sd 0:0:176:0: [sdfr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294414] sd 0:0:176:0: [sdfr] 4096-byte physical blocks [ 39.294426] sd 0:0:131:0: [sddy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294427] sd 0:0:131:0: [sddy] 4096-byte physical blocks [ 39.294448] sd 0:0:129:0: [sddw] Write Protect is off [ 39.294449] sd 0:0:129:0: [sddw] Mode Sense: db 00 10 08 [ 39.294536] sd 0:0:139:0: [sdeg] Write Protect is off [ 39.294538] sd 0:0:139:0: [sdeg] Mode Sense: db 00 10 08 [ 39.294610] sd 0:0:132:0: [sddz] Write Protect is off [ 39.294611] sd 0:0:132:0: [sddz] Mode Sense: db 00 10 08 [ 39.294627] sd 0:0:167:0: [sdfi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294628] sd 0:0:167:0: [sdfi] 4096-byte physical blocks [ 39.294657] sd 0:0:144:0: [sdel] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294659] sd 0:0:144:0: [sdel] 4096-byte physical blocks [ 39.294725] sd 0:0:136:0: [sded] Write Protect is off [ 39.294726] sd 0:0:104:0: [sdcy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294727] sd 0:0:136:0: [sded] Mode Sense: db 00 10 08 [ 39.294727] sd 0:0:104:0: [sdcy] 4096-byte physical blocks [ 39.294745] sd 0:0:2:0: [sdb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.294750] sd 0:0:134:0: [sdeb] Write Protect is off [ 39.294752] sd 0:0:134:0: [sdeb] Mode Sense: db 00 10 08 [ 39.294757] sd 0:0:42:0: [sdap] Write Protect is off [ 39.294758] sd 0:0:42:0: [sdap] Mode Sense: db 00 10 08 [ 39.294769] sd 0:0:40:0: [sdan] Write Protect is off [ 39.294771] sd 0:0:40:0: [sdan] Mode Sense: db 00 10 08 [ 39.294788] sd 0:0:137:0: [sdee] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.294878] sd 0:0:138:0: [sdef] Write Protect is off [ 39.294880] sd 0:0:138:0: [sdef] Mode Sense: db 00 10 08 [ 39.294931] sd 0:0:43:0: [sdaq] Write Protect is off [ 39.294933] sd 0:0:43:0: [sdaq] Mode Sense: db 00 10 08 [ 39.294942] sd 0:0:113:0: [sddh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.294944] sd 0:0:113:0: [sddh] 4096-byte physical blocks [ 39.295016] sd 0:0:141:0: [sdei] Write Protect is off [ 39.295018] sd 0:0:141:0: [sdei] Mode Sense: db 00 10 08 [ 39.295112] sd 0:0:149:0: [sdeq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295113] sd 0:0:149:0: [sdeq] 4096-byte physical blocks [ 39.295285] sd 0:0:147:0: [sdeo] Write Protect is off [ 39.295286] sd 0:0:147:0: [sdeo] Mode Sense: db 00 10 08 [ 39.295355] sd 0:0:133:0: [sdea] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.295356] sd 0:0:87:0: [sdch] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295358] sd 0:0:87:0: [sdch] 4096-byte physical blocks [ 39.295365] sd 0:0:143:0: [sdek] Write Protect is off [ 39.295366] sd 0:0:143:0: [sdek] Mode Sense: db 00 10 08 [ 39.295407] sd 0:0:80:0: [sdca] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295408] sd 0:0:80:0: [sdca] 4096-byte physical blocks [ 39.295474] sd 0:0:108:0: [sddc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295476] sd 0:0:108:0: [sddc] 4096-byte physical blocks [ 39.295505] sd 0:0:142:0: [sdej] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295507] sd 0:0:142:0: [sdej] 4096-byte physical blocks [ 39.295531] sd 0:0:53:0: [sdba] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295532] sd 0:0:53:0: [sdba] 4096-byte physical blocks [ 39.295550] sd 0:0:146:0: [sden] Write Protect is off [ 39.295552] sd 0:0:146:0: [sden] Mode Sense: db 00 10 08 [ 39.295580] sd 0:0:145:0: [sdem] Write Protect is off [ 39.295581] sd 0:0:145:0: [sdem] Mode Sense: db 00 10 08 [ 39.295618] sd 0:0:30:0: [sdad] Write Protect is off [ 39.295620] sd 0:0:30:0: [sdad] Mode Sense: db 00 10 08 [ 39.295775] sd 0:0:129:0: [sddw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.295794] sd 0:0:198:0: [sdgm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295796] sd 0:0:198:0: [sdgm] 4096-byte physical blocks [ 39.295804] sd 0:0:139:0: [sdeg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.295805] sd 0:0:55:0: [sdbc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295807] sd 0:0:55:0: [sdbc] 4096-byte physical blocks [ 39.295832] sd 0:0:28:0: [sdab] Write Protect is off [ 39.295833] sd 0:0:28:0: [sdab] Mode Sense: db 00 10 08 [ 39.295862] sd 0:0:175:0: [sdfq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295864] sd 0:0:175:0: [sdfq] 4096-byte physical blocks [ 39.295899] sd 0:0:29:0: [sdac] Write Protect is off [ 39.295900] sd 0:0:29:0: [sdac] Mode Sense: db 00 10 08 [ 39.295903] sd 0:0:77:0: [sdbx] Write Protect is off [ 39.295904] sd 0:0:77:0: [sdbx] Mode Sense: db 00 10 08 [ 39.295927] sd 0:0:125:0: [sdds] Write Protect is off [ 39.295928] sd 0:0:125:0: [sdds] Mode Sense: db 00 10 08 [ 39.295941] sd 0:0:73:0: [sdbt] Write Protect is off [ 39.295943] sd 0:0:73:0: [sdbt] Mode Sense: db 00 10 08 [ 39.295954] sd 0:0:123:0: [sddq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.295962] sd 0:0:140:0: [sdeh] Write Protect is off [ 39.295964] sd 0:0:140:0: [sdeh] Mode Sense: db 00 10 08 [ 39.295972] sd 0:0:44:0: [sdar] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.295973] sd 0:0:44:0: [sdar] 4096-byte physical blocks [ 39.296009] sd 0:0:111:0: [sddf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296010] sd 0:0:111:0: [sddf] 4096-byte physical blocks [ 39.296035] sd 0:0:21:0: [sdu] Write Protect is off [ 39.296038] sd 0:0:54:0: [sdbb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296040] sd 0:0:21:0: [sdu] Mode Sense: db 00 10 08 [ 39.296062] sd 0:0:90:0: [sdck] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296063] sd 0:0:124:0: [sddr] Write Protect is off [ 39.296064] sd 0:0:90:0: [sdck] 4096-byte physical blocks [ 39.296065] sd 0:0:124:0: [sddr] Mode Sense: db 00 10 08 [ 39.296072] sd 0:0:136:0: [sded] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296079] sd 0:0:126:0: [sddt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296090] sd 0:0:118:0: [sddm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296091] sd 0:0:118:0: [sddm] 4096-byte physical blocks [ 39.296112] sd 0:0:62:0: [sdbi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296113] sd 0:0:62:0: [sdbi] 4096-byte physical blocks [ 39.296116] sd 0:0:94:0: [sdco] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296117] sd 0:0:94:0: [sdco] 4096-byte physical blocks [ 39.296136] sd 0:0:177:0: [sdfs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296138] sd 0:0:177:0: [sdfs] 4096-byte physical blocks [ 39.296197] sd 0:0:202:0: [sdgq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296198] sd 0:0:202:0: [sdgq] 4096-byte physical blocks [ 39.296204] sd 0:0:128:0: [sddv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296215] sd 0:0:24:0: [sdx] Write Protect is off [ 39.296216] sd 0:0:24:0: [sdx] Mode Sense: db 00 10 08 [ 39.296228] sd 0:0:10:0: [sdj] Write Protect is off [ 39.296229] sd 0:0:10:0: [sdj] Mode Sense: db 00 10 08 [ 39.296254] sd 0:0:199:0: [sdgn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296255] sd 0:0:199:0: [sdgn] 4096-byte physical blocks [ 39.296263] sd 0:0:23:0: [sdw] Write Protect is off [ 39.296264] sd 0:0:23:0: [sdw] Mode Sense: db 00 10 08 [ 39.296266] sd 0:0:130:0: [sddx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296268] sd 0:0:127:0: [sddu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296323] sd 0:0:166:0: [sdfh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296325] sd 0:0:166:0: [sdfh] 4096-byte physical blocks [ 39.296336] sd 0:0:96:0: [sdcq] Write Protect is off [ 39.296338] sd 0:0:96:0: [sdcq] Mode Sense: db 00 10 08 [ 39.296347] sd 0:0:46:0: [sdat] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296349] sd 0:0:135:0: [sdec] Write Protect is off [ 39.296350] sd 0:0:135:0: [sdec] Mode Sense: db 00 10 08 [ 39.296395] sd 0:0:12:0: [sdl] Write Protect is off [ 39.296396] sd 0:0:12:0: [sdl] Mode Sense: db 00 10 08 [ 39.296451] sd 0:0:184:0: [sdfy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296453] sd 0:0:184:0: [sdfy] 4096-byte physical blocks [ 39.296469] sd 0:0:150:0: [sder] Write Protect is off [ 39.296470] sd 0:0:150:0: [sder] Mode Sense: db 00 10 08 [ 39.296476] sd 0:0:86:0: [sdcg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296478] sd 0:0:86:0: [sdcg] 4096-byte physical blocks [ 39.296479] sd 0:0:121:0: [sddp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296485] sd 0:0:121:0: [sddp] 4096-byte physical blocks [ 39.296527] sd 0:0:16:0: [sdp] Write Protect is off [ 39.296528] sd 0:0:16:0: [sdp] Mode Sense: db 00 10 08 [ 39.296553] sd 0:0:209:0: [sdgx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296556] sd 0:0:3:0: [sdc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296558] sd 0:0:209:0: [sdgx] 4096-byte physical blocks [ 39.296576] sd 0:0:151:0: [sdes] Write Protect is off [ 39.296578] sd 0:0:151:0: [sdes] Mode Sense: db 00 10 08 [ 39.296595] sd 0:0:1:0: [sda] Write Protect is off [ 39.296597] sd 0:0:1:0: [sda] Mode Sense: db 00 10 08 [ 39.296614] sd 0:0:147:0: [sdeo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296630] sd 0:0:19:0: [sds] Write Protect is off [ 39.296632] sd 0:0:19:0: [sds] Mode Sense: db 00 10 08 [ 39.296638] sd 0:0:63:0: [sdbj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296639] sd 0:0:63:0: [sdbj] 4096-byte physical blocks [ 39.296704] sd 0:0:131:0: [sddy] Write Protect is off [ 39.296706] sd 0:0:131:0: [sddy] Mode Sense: db 00 10 08 [ 39.296732] sd 0:0:4:0: [sdd] Write Protect is off [ 39.296734] sd 0:0:4:0: [sdd] Mode Sense: db 00 10 08 [ 39.296744] sd 0:0:42:0: [sdap] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296781] sd 0:0:40:0: [sdan] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296811] sd 0:0:208:0: [sdgw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296813] sd 0:0:208:0: [sdgw] 4096-byte physical blocks [ 39.296830] sd 0:0:119:0: [sddn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296831] sd 0:0:119:0: [sddn] 4096-byte physical blocks [ 39.296841] sd 0:0:6:0: [sdf] Write Protect is off [ 39.296843] sd 0:0:6:0: [sdf] Mode Sense: db 00 10 08 [ 39.296847] sd 0:0:212:0: [sdha] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296849] sd 0:0:212:0: [sdha] 4096-byte physical blocks [ 39.296869] sd 0:0:132:0: [sddz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296880] sd 0:0:106:0: [sdda] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296882] sd 0:0:106:0: [sdda] 4096-byte physical blocks [ 39.296906] sd 0:0:43:0: [sdaq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.296927] sd 0:0:8:0: [sdh] Write Protect is off [ 39.296928] sd 0:0:8:0: [sdh] Mode Sense: db 00 10 08 [ 39.296938] sd 0:0:5:0: [sde] Write Protect is off [ 39.296940] sd 0:0:5:0: [sde] Mode Sense: db 00 10 08 [ 39.296942] sd 0:0:216:0: [sdhe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296943] sd 0:0:216:0: [sdhe] 4096-byte physical blocks [ 39.296956] sd 0:0:195:0: [sdgj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296957] sd 0:0:195:0: [sdgj] 4096-byte physical blocks [ 39.296959] sd 0:0:60:0: [sdbh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296962] sd 0:0:60:0: [sdbh] 4096-byte physical blocks [ 39.296968] sd 0:0:222:0: [sdhk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.296970] sd 0:0:222:0: [sdhk] 4096-byte physical blocks [ 39.297004] sd 0:0:134:0: [sdeb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297035] sd 0:0:185:0: [sdfz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297036] sd 0:0:185:0: [sdfz] 4096-byte physical blocks [ 39.297050] sd 0:0:144:0: [sdel] Write Protect is off [ 39.297051] sd 0:0:144:0: [sdel] Mode Sense: db 00 10 08 [ 39.297062] sd 0:0:104:0: [sdcy] Write Protect is off [ 39.297064] sd 0:0:104:0: [sdcy] Mode Sense: db 00 10 08 [ 39.297154] sd 0:0:138:0: [sdef] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297170] sd 0:0:113:0: [sddh] Write Protect is off [ 39.297171] sd 0:0:113:0: [sddh] Mode Sense: db 00 10 08 [ 39.297248] sd 0:0:141:0: [sdei] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297259] sd 0:0:152:0: [sdet] Write Protect is off [ 39.297261] sd 0:0:152:0: [sdet] Mode Sense: db 00 10 08 [ 39.297296] sd 0:0:182:0: [sdfx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297297] sd 0:0:182:0: [sdfx] 4096-byte physical blocks [ 39.297307] sd 0:0:140:0: [sdeh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297317] sd 0:0:223:0: [sdhl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297318] sd 0:0:223:0: [sdhl] 4096-byte physical blocks [ 39.297339] sd 0:0:149:0: [sdeq] Write Protect is off [ 39.297340] sd 0:0:149:0: [sdeq] Mode Sense: db 00 10 08 [ 39.297393] sd 0:0:69:0: [sdbp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297394] sd 0:0:69:0: [sdbp] 4096-byte physical blocks [ 39.297440] sd 0:0:218:0: [sdhg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297441] sd 0:0:218:0: [sdhg] 4096-byte physical blocks [ 39.297467] sd 0:0:112:0: [sddg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297469] sd 0:0:112:0: [sddg] 4096-byte physical blocks [ 39.297521] sd 0:0:71:0: [sdbr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297522] sd 0:0:71:0: [sdbr] 4096-byte physical blocks [ 39.297555] sd 0:0:48:0: [sdav] Write Protect is off [ 39.297556] sd 0:0:48:0: [sdav] Mode Sense: db 00 10 08 [ 39.297562] sd 0:0:73:0: [sdbt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297577] sd 0:0:180:0: [sdfv] Write Protect is off [ 39.297578] sd 0:0:180:0: [sdfv] Mode Sense: db 00 10 08 [ 39.297644] sd 0:0:135:0: [sdec] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297652] sd 0:0:67:0: [sdbn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297653] sd 0:0:67:0: [sdbn] 4096-byte physical blocks [ 39.297741] sd 0:0:191:0: [sdgf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297743] sd 0:0:191:0: [sdgf] 4096-byte physical blocks [ 39.297759] sd 0:0:228:0: [sdhq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297760] sd 0:0:228:0: [sdhq] 4096-byte physical blocks [ 39.297790] sd 0:0:108:0: [sddc] Write Protect is off [ 39.297792] sd 0:0:108:0: [sddc] Mode Sense: db 00 10 08 [ 39.297814] sd 0:0:49:0: [sdaw] Write Protect is off [ 39.297815] sd 0:0:49:0: [sdaw] Mode Sense: db 00 10 08 [ 39.297825] sd 0:0:142:0: [sdej] Write Protect is off [ 39.297826] sd 0:0:142:0: [sdej] Mode Sense: db 00 10 08 [ 39.297837] sd 0:0:47:0: [sdau] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297839] sd 0:0:47:0: [sdau] 4096-byte physical blocks [ 39.297851] sd 0:0:193:0: [sdgh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297853] sd 0:0:193:0: [sdgh] 4096-byte physical blocks [ 39.297862] sd 0:0:85:0: [sdcf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297864] sd 0:0:85:0: [sdcf] 4096-byte physical blocks [ 39.297884] sd 0:0:30:0: [sdad] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297912] sd 0:0:161:0: [sdfc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297914] sd 0:0:161:0: [sdfc] 4096-byte physical blocks [ 39.297922] sd 0:0:77:0: [sdbx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297923] sd 0:0:143:0: [sdek] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297943] sd 0:0:70:0: [sdbq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297944] sd 0:0:70:0: [sdbq] 4096-byte physical blocks [ 39.297988] sd 0:0:153:0: [sdeu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.297989] sd 0:0:153:0: [sdeu] 4096-byte physical blocks [ 39.297991] sd 0:0:131:0: [sddy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.297999] sd 0:0:189:0: [sdgd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298001] sd 0:0:189:0: [sdgd] 4096-byte physical blocks [ 39.298025] sd 0:0:124:0: [sddr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298042] sd 0:0:45:0: [sdas] Write Protect is off [ 39.298044] sd 0:0:45:0: [sdas] Mode Sense: db 00 10 08 [ 39.298059] sd 0:0:96:0: [sdcq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298066] sd 0:0:146:0: [sden] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298086] sd 0:0:164:0: [sdff] Write Protect is off [ 39.298087] sd 0:0:164:0: [sdff] Mode Sense: db 00 10 08 [ 39.298120] sd 0:0:145:0: [sdem] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298126] sd 0:0:207:0: [sdgv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298128] sd 0:0:207:0: [sdgv] 4096-byte physical blocks [ 39.298136] sd 0:0:162:0: [sdfd] Write Protect is off [ 39.298137] sd 0:0:162:0: [sdfd] Mode Sense: db 00 10 08 [ 39.298151] sd 0:0:125:0: [sdds] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298159] sd 0:0:169:0: [sdfk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298160] sd 0:0:169:0: [sdfk] 4096-byte physical blocks [ 39.298239] sd 0:0:170:0: [sdfl] Write Protect is off [ 39.298240] sd 0:0:170:0: [sdfl] Mode Sense: db 00 10 08 [ 39.298245] sd 0:0:111:0: [sddf] Write Protect is off [ 39.298246] sd 0:0:111:0: [sddf] Mode Sense: db 00 10 08 [ 39.298256] sd 0:0:165:0: [sdfg] Write Protect is off [ 39.298257] sd 0:0:165:0: [sdfg] Mode Sense: db 00 10 08 [ 39.298280] sd 0:0:192:0: [sdgg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298301] sd 0:0:192:0: [sdgg] 4096-byte physical blocks [ 39.298352] sd 0:0:168:0: [sdfj] Write Protect is off [ 39.298353] sd 0:0:168:0: [sdfj] Mode Sense: db 00 10 08 [ 39.298360] sd 0:0:28:0: [sdab] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298362] sd 0:0:144:0: [sdel] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298363] sd 0:0:118:0: [sddm] Write Protect is off [ 39.298364] sd 0:0:118:0: [sddm] Mode Sense: db 00 10 08 [ 39.298417] sd 0:0:29:0: [sdac] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298419] sd 0:0:104:0: [sdcy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298435] sd 0:0:171:0: [sdfm] Write Protect is off [ 39.298436] sd 0:0:171:0: [sdfm] Mode Sense: db 00 10 08 [ 39.298456] sd 0:0:10:0: [sdj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298507] sd 0:0:110:0: [sdde] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298509] sd 0:0:110:0: [sdde] 4096-byte physical blocks [ 39.298532] sd 1:2:0:0: [sdig] 233308160 512-byte logical blocks: (119 GB/111 GiB) [ 39.298542] sd 0:0:1:0: [sda] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298608] sd 0:0:113:0: [sddh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298616] sd 0:0:87:0: [sdch] Write Protect is off [ 39.298617] sd 0:0:87:0: [sdch] Mode Sense: db 00 10 08 [ 39.298630] sd 0:0:21:0: [sdu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298640] sd 0:0:12:0: [sdl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298651] sd 0:0:99:0: [sdct] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298652] sd 0:0:99:0: [sdct] 4096-byte physical blocks [ 39.298672] sd 0:0:167:0: [sdfi] Write Protect is off [ 39.298674] sd 0:0:167:0: [sdfi] Mode Sense: db 00 10 08 [ 39.298690] sd 0:0:4:0: [sdd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298700] sd 0:0:234:0: [sdhw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298701] sd 0:0:107:0: [sddb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298702] sd 0:0:234:0: [sdhw] 4096-byte physical blocks [ 39.298702] sd 0:0:107:0: [sddb] 4096-byte physical blocks [ 39.298708] sd 0:0:149:0: [sdeq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298725] sd 0:0:16:0: [sdp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298743] sd 0:0:24:0: [sdx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298765] sd 0:0:23:0: [sdw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298770] sd 0:0:176:0: [sdfr] Write Protect is off [ 39.298772] sd 0:0:176:0: [sdfr] Mode Sense: db 00 10 08 [ 39.298794] sd 0:0:121:0: [sddp] Write Protect is off [ 39.298796] sd 0:0:226:0: [sdho] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298796] sd 0:0:121:0: [sddp] Mode Sense: db 00 10 08 [ 39.298797] sd 0:0:226:0: [sdho] 4096-byte physical blocks [ 39.298801] sd 0:0:6:0: [sdf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298830] sd 0:0:19:0: [sds] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298857] sd 0:0:232:0: [sdhu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298858] sd 0:0:232:0: [sdhu] 4096-byte physical blocks [ 39.298872] sd 0:0:5:0: [sde] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298905] sd 0:0:26:0: [sdz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298906] sd 0:0:26:0: [sdz] 4096-byte physical blocks [ 39.298910] sd 0:0:8:0: [sdh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298919] sd 0:0:231:0: [sdht] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298921] sd 0:0:231:0: [sdht] 4096-byte physical blocks [ 39.298926] sd 0:0:221:0: [sdhj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298928] sd 0:0:221:0: [sdhj] 4096-byte physical blocks [ 39.298936] sd 0:0:80:0: [sdca] Write Protect is off [ 39.298938] sd 0:0:80:0: [sdca] Mode Sense: db 00 10 08 [ 39.298946] sd 0:0:230:0: [sdhs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298947] sd 0:0:230:0: [sdhs] 4096-byte physical blocks [ 39.298972] sd 0:0:150:0: [sder] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.298997] sd 0:0:235:0: [sdhx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.298998] sd 0:0:235:0: [sdhx] 4096-byte physical blocks [ 39.299000] sd 0:0:74:0: [sdbu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299002] sd 0:0:74:0: [sdbu] 4096-byte physical blocks [ 39.299077] sd 1:2:0:0: [sdig] Write Protect is off [ 39.299079] sd 1:2:0:0: [sdig] Mode Sense: 1f 00 10 08 [ 39.299090] sd 0:0:82:0: [sdcc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299092] sd 0:0:82:0: [sdcc] 4096-byte physical blocks [ 39.299109] sd 0:0:41:0: [sdao] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299111] sd 0:0:41:0: [sdao] 4096-byte physical blocks [ 39.299134] sd 0:0:229:0: [sdhr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299135] sd 0:0:229:0: [sdhr] 4096-byte physical blocks [ 39.299137] sd 1:2:0:0: [sdig] Write cache: disabled, read cache: disabled, supports DPO and FUA [ 39.299144] sd 0:0:151:0: [sdes] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.299155] sd 0:0:239:0: [sdib] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299156] sd 0:0:239:0: [sdib] 4096-byte physical blocks [ 39.299215] sd 0:0:238:0: [sdia] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299216] sd 0:0:238:0: [sdia] 4096-byte physical blocks [ 39.299238] sd 0:0:142:0: [sdej] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.299239] sd 0:0:119:0: [sddn] Write Protect is off [ 39.299241] sd 0:0:119:0: [sddn] Mode Sense: db 00 10 08 [ 39.299257] sd 0:0:53:0: [sdba] Write Protect is off [ 39.299259] sd 0:0:53:0: [sdba] Mode Sense: db 00 10 08 [ 39.299264] sd 0:0:148:0: [sdep] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299265] sd 0:0:148:0: [sdep] 4096-byte physical blocks [ 39.299329] sd 0:0:233:0: [sdhv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299331] sd 0:0:233:0: [sdhv] 4096-byte physical blocks [ 39.299398] sd 0:0:204:0: [sdgs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299399] sd 0:0:204:0: [sdgs] 4096-byte physical blocks [ 39.299429] sd 0:0:163:0: [sdfe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299430] sd 0:0:163:0: [sdfe] 4096-byte physical blocks [ 39.299503] sd 0:0:240:0: [sdic] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299505] sd 0:0:240:0: [sdic] 4096-byte physical blocks [ 39.299507] sd 0:0:152:0: [sdet] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.299536] sd 0:0:108:0: [sddc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.299567] sd 0:0:90:0: [sdck] Write Protect is off [ 39.299569] sd 0:0:90:0: [sdck] Mode Sense: db 00 10 08 [ 39.299594] sd 0:0:68:0: [sdbo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299596] sd 0:0:68:0: [sdbo] 4096-byte physical blocks [ 39.299612] sd 0:0:66:0: [sdbm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299613] sd 0:0:66:0: [sdbm] 4096-byte physical blocks [ 39.299622] sd 0:0:241:0: [sdid] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299623] sd 0:0:241:0: [sdid] 4096-byte physical blocks [ 39.299649] sd 0:0:94:0: [sdco] Write Protect is off [ 39.299651] sd 0:0:94:0: [sdco] Mode Sense: db 00 10 08 [ 39.299666] sd 0:0:84:0: [sdce] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299668] sd 0:0:84:0: [sdce] 4096-byte physical blocks [ 39.299669] sd 0:0:196:0: [sdgk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299670] sd 0:0:196:0: [sdgk] 4096-byte physical blocks [ 39.299857] sd 0:0:62:0: [sdbi] Write Protect is off [ 39.299859] sd 0:0:62:0: [sdbi] Mode Sense: db 00 10 08 [ 39.299872] sd 0:0:75:0: [sdbv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299873] sd 0:0:75:0: [sdbv] 4096-byte physical blocks [ 39.299951] sd 0:0:55:0: [sdbc] Write Protect is off [ 39.299953] sd 0:0:55:0: [sdbc] Mode Sense: db 00 10 08 [ 39.299983] sd 0:0:18:0: [sdr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.299984] sd 0:0:44:0: [sdar] Write Protect is off [ 39.299987] sd 0:0:44:0: [sdar] Mode Sense: db 00 10 08 [ 39.299989] sd 0:0:18:0: [sdr] 4096-byte physical blocks [ 39.300002] sd 0:0:25:0: [sdy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300004] sd 0:0:25:0: [sdy] 4096-byte physical blocks [ 39.300036] sd 0:0:27:0: [sdaa] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300037] sd 0:0:27:0: [sdaa] 4096-byte physical blocks [ 39.300044] sd 0:0:22:0: [sdv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300046] sd 0:0:22:0: [sdv] 4096-byte physical blocks [ 39.300070] sd 0:0:111:0: [sddf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300082] sd 0:0:243:0: [sdif] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300084] sd 0:0:243:0: [sdif] 4096-byte physical blocks [ 39.300088] sd 0:0:188:0: [sdgc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300089] sd 0:0:188:0: [sdgc] 4096-byte physical blocks [ 39.300111] sd 0:0:190:0: [sdge] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300112] sd 0:0:162:0: [sdfd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300114] sd 0:0:190:0: [sdge] 4096-byte physical blocks [ 39.300155] sd 0:0:168:0: [sdfj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300174] sd 0:0:159:0: [sdfa] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300176] sd 0:0:159:0: [sdfa] 4096-byte physical blocks [ 39.300181] sd 0:0:164:0: [sdff] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300195] sd 0:0:175:0: [sdfq] Write Protect is off [ 39.300197] sd 0:0:175:0: [sdfq] Mode Sense: db 00 10 08 [ 39.300228] sd 0:0:118:0: [sddm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300247] sd 0:0:180:0: [sdfv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300277] sd 0:0:206:0: [sdgu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300279] sd 0:0:206:0: [sdgu] 4096-byte physical blocks [ 39.300281] sd 0:0:197:0: [sdgl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300282] sd 0:0:197:0: [sdgl] 4096-byte physical blocks [ 39.300297] sd 0:0:86:0: [sdcg] Write Protect is off [ 39.300299] sd 0:0:86:0: [sdcg] Mode Sense: db 00 10 08 [ 39.300301] sd 0:0:79:0: [sdbz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300303] sd 0:0:79:0: [sdbz] 4096-byte physical blocks [ 39.300329] sd 0:0:165:0: [sdfg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300350] sd 0:0:177:0: [sdfs] Write Protect is off [ 39.300352] sd 0:0:177:0: [sdfs] Mode Sense: db 00 10 08 [ 39.300353] sd 0:0:161:0: [sdfc] Write Protect is off [ 39.300355] sd 0:0:161:0: [sdfc] Mode Sense: db 00 10 08 [ 39.300400] sd 0:0:166:0: [sdfh] Write Protect is off [ 39.300402] sd 0:0:166:0: [sdfh] Mode Sense: db 00 10 08 [ 39.300425] sd 0:0:20:0: [sdt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300427] sd 0:0:20:0: [sdt] 4096-byte physical blocks [ 39.300430] sd 0:0:13:0: [sdm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300431] sd 0:0:63:0: [sdbj] Write Protect is off [ 39.300433] sd 0:0:13:0: [sdm] 4096-byte physical blocks [ 39.300434] sd 0:0:63:0: [sdbj] Mode Sense: db 00 10 08 [ 39.300523] sd 0:0:48:0: [sdav] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300538] sd 0:0:9:0: [sdi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300539] sd 0:0:9:0: [sdi] 4096-byte physical blocks [ 39.300546] sd 0:0:106:0: [sdda] Write Protect is off [ 39.300548] sd 0:0:106:0: [sdda] Mode Sense: db 00 10 08 [ 39.300574] sd 0:0:184:0: [sdfy] Write Protect is off [ 39.300576] sd 0:0:184:0: [sdfy] Mode Sense: db 00 10 08 [ 39.300617] sd 0:0:87:0: [sdch] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300653] sd 0:0:176:0: [sdfr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300662] sd 0:0:179:0: [sdfu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300664] sd 0:0:179:0: [sdfu] 4096-byte physical blocks [ 39.300669] sd 0:0:201:0: [sdgp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300671] sd 0:0:201:0: [sdgp] 4096-byte physical blocks [ 39.300704] sd 0:0:121:0: [sddp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300712] sd 0:0:199:0: [sdgn] Write Protect is off [ 39.300713] sd 0:0:199:0: [sdgn] Mode Sense: db 00 10 08 [ 39.300727] sd 0:0:198:0: [sdgm] Write Protect is off [ 39.300729] sd 0:0:198:0: [sdgm] Mode Sense: db 00 10 08 [ 39.300731] sd 0:0:170:0: [sdfl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300750] sd 0:0:202:0: [sdgq] Write Protect is off [ 39.300751] sd 0:0:202:0: [sdgq] Mode Sense: db 00 10 08 [ 39.300755] sd 0:0:57:0: [sdbe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300758] sd 0:0:7:0: [sdg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300760] sd 0:0:57:0: [sdbe] 4096-byte physical blocks [ 39.300762] sd 0:0:7:0: [sdg] 4096-byte physical blocks [ 39.300804] sd 0:0:64:0: [sdbk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300806] sd 0:0:64:0: [sdbk] 4096-byte physical blocks [ 39.300814] sdig: sdig1 sdig2 sdig3 [ 39.300820] sd 0:0:31:0: [sdae] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300823] sd 0:0:31:0: [sdae] 4096-byte physical blocks [ 39.300830] sd 0:0:56:0: [sdbd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300832] sd 0:0:56:0: [sdbd] 4096-byte physical blocks [ 39.300840] sd 0:0:49:0: [sdaw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.300858] sd 0:0:14:0: [sdn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300860] sd 0:0:14:0: [sdn] 4096-byte physical blocks [ 39.300864] sd 0:0:172:0: [sdfn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300866] sd 0:0:172:0: [sdfn] 4096-byte physical blocks [ 39.300873] sd 0:0:17:0: [sdq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300875] sd 0:0:17:0: [sdq] 4096-byte physical blocks [ 39.300876] sd 0:0:178:0: [sdft] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300877] sd 0:0:178:0: [sdft] 4096-byte physical blocks [ 39.300902] sd 0:0:112:0: [sddg] Write Protect is off [ 39.300904] sd 0:0:112:0: [sddg] Mode Sense: db 00 10 08 [ 39.300935] sd 0:0:195:0: [sdgj] Write Protect is off [ 39.300936] sd 0:0:195:0: [sdgj] Mode Sense: db 00 10 08 [ 39.300940] sd 0:0:208:0: [sdgw] Write Protect is off [ 39.300941] sd 0:0:208:0: [sdgw] Mode Sense: db 00 10 08 [ 39.300963] sd 0:0:186:0: [sdga] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300965] sd 0:0:186:0: [sdga] 4096-byte physical blocks [ 39.300994] sd 0:0:11:0: [sdk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.300996] sd 0:0:11:0: [sdk] 4096-byte physical blocks [ 39.301034] sd 0:0:153:0: [sdeu] Write Protect is off [ 39.301036] sd 0:0:153:0: [sdeu] Mode Sense: db 00 10 08 [ 39.301044] sd 0:0:171:0: [sdfm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.301062] sd 0:0:91:0: [sdcl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.301064] sd 0:0:91:0: [sdcl] 4096-byte physical blocks [ 39.301065] sd 0:0:45:0: [sdas] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.301070] sd 0:0:115:0: [sddj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.301071] sd 0:0:115:0: [sddj] 4096-byte physical blocks [ 39.301106] sd 0:0:185:0: [sdfz] Write Protect is off [ 39.301108] sd 0:0:185:0: [sdfz] Mode Sense: db 00 10 08 [ 39.301231] sd 0:0:69:0: [sdbp] Write Protect is off [ 39.301233] sd 0:0:69:0: [sdbp] Mode Sense: db 00 10 08 [ 39.301244] sd 0:0:237:0: [sdhz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.301245] sd 0:0:237:0: [sdhz] 4096-byte physical blocks [ 39.301261] sd 0:0:119:0: [sddn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.301280] sd 0:0:60:0: [sdbh] Write Protect is off [ 39.301281] sd 0:0:60:0: [sdbh] Mode Sense: db 00 10 08 [ 39.301290] sd 0:0:167:0: [sdfi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.301320] sd 0:0:97:0: [sdcr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.301322] sd 0:0:97:0: [sdcr] 4096-byte physical blocks [ 39.301332] sd 0:0:80:0: [sdca] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.301364] sd 0:0:209:0: [sdgx] Write Protect is off [ 39.301365] sd 0:0:209:0: [sdgx] Mode Sense: db 00 10 08 [ 39.301368] sd 0:0:71:0: [sdbr] Write Protect is off [ 39.301369] sd 0:0:71:0: [sdbr] Mode Sense: db 00 10 08 [ 39.301379] sd 0:0:218:0: [sdhg] Write Protect is off [ 39.301380] sd 0:0:218:0: [sdhg] Mode Sense: db 00 10 08 [ 39.301387] sd 0:0:216:0: [sdhe] Write Protect is off [ 39.301388] sd 0:0:216:0: [sdhe] Mode Sense: db 00 10 08 [ 39.301389] sd 0:0:212:0: [sdha] Write Protect is off [ 39.301391] sd 0:0:212:0: [sdha] Mode Sense: db 00 10 08 [ 39.301399] sd 0:0:213:0: [sdhb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.301401] sd 0:0:213:0: [sdhb] 4096-byte physical blocks [ 39.301423] sd 0:0:67:0: [sdbn] Write Protect is off [ 39.301424] sd 0:0:67:0: [sdbn] Mode Sense: db 00 10 08 [ 39.301512] sd 0:0:219:0: [sdhh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.301513] sd 0:0:219:0: [sdhh] 4096-byte physical blocks [ 39.301544] sd 1:2:0:0: [sdig] Attached SCSI disk [ 39.301587] sd 0:0:50:0: [sdax] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.301588] sd 0:0:50:0: [sdax] 4096-byte physical blocks [ 39.301632] sd 0:0:85:0: [sdcf] Write Protect is off [ 39.301634] sd 0:0:85:0: [sdcf] Mode Sense: db 00 10 08 [ 39.301816] sd 0:0:222:0: [sdhk] Write Protect is off [ 39.301818] sd 0:0:222:0: [sdhk] Mode Sense: db 00 10 08 [ 39.301896] sd 0:0:191:0: [sdgf] Write Protect is off [ 39.301898] sd 0:0:191:0: [sdgf] Mode Sense: db 00 10 08 [ 39.302012] sd 0:0:193:0: [sdgh] Write Protect is off [ 39.302013] sd 0:0:193:0: [sdgh] Mode Sense: db 00 10 08 [ 39.302023] sd 0:0:223:0: [sdhl] Write Protect is off [ 39.302025] sd 0:0:223:0: [sdhl] Mode Sense: db 00 10 08 [ 39.302066] sd 0:0:93:0: [sdcn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302067] sd 0:0:93:0: [sdcn] 4096-byte physical blocks [ 39.302096] sd 0:0:52:0: [sdaz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302098] sd 0:0:52:0: [sdaz] 4096-byte physical blocks [ 39.302107] sd 0:0:174:0: [sdfp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302108] sd 0:0:174:0: [sdfp] 4096-byte physical blocks [ 39.302120] sd 0:0:70:0: [sdbq] Write Protect is off [ 39.302122] sd 0:0:70:0: [sdbq] Mode Sense: db 00 10 08 [ 39.302131] sd 0:0:94:0: [sdco] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302173] sd 0:0:90:0: [sdck] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302276] sd 0:0:228:0: [sdhq] Write Protect is off [ 39.302278] sd 0:0:228:0: [sdhq] Mode Sense: db 00 10 08 [ 39.302375] sd 0:0:189:0: [sdgd] Write Protect is off [ 39.302376] sd 0:0:189:0: [sdgd] Mode Sense: db 00 10 08 [ 39.302389] sd 0:0:215:0: [sdhd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302390] sd 0:0:215:0: [sdhd] 4096-byte physical blocks [ 39.302399] sd 0:0:207:0: [sdgv] Write Protect is off [ 39.302401] sd 0:0:207:0: [sdgv] Mode Sense: db 00 10 08 [ 39.302425] sd 0:0:173:0: [sdfo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302426] sd 0:0:173:0: [sdfo] 4096-byte physical blocks [ 39.302434] sd 0:0:182:0: [sdfx] Write Protect is off [ 39.302435] sd 0:0:182:0: [sdfx] Mode Sense: db 00 10 08 [ 39.302494] sd 0:0:53:0: [sdba] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302577] sd 0:0:55:0: [sdbc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302599] sd 0:0:192:0: [sdgg] Write Protect is off [ 39.302600] sd 0:0:192:0: [sdgg] Mode Sense: db 00 10 08 [ 39.302604] sd 0:0:44:0: [sdar] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302630] sd 0:0:99:0: [sdct] Write Protect is off [ 39.302631] sd 0:0:99:0: [sdct] Mode Sense: db 00 10 08 [ 39.302637] sd 0:0:198:0: [sdgm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302670] sd 0:0:51:0: [sday] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302672] sd 0:0:51:0: [sday] 4096-byte physical blocks [ 39.302695] sd 0:0:199:0: [sdgn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302844] sd 0:0:235:0: [sdhx] Write Protect is off [ 39.302846] sd 0:0:235:0: [sdhx] Mode Sense: db 00 10 08 [ 39.302863] sd 0:0:110:0: [sdde] Write Protect is off [ 39.302864] sd 0:0:110:0: [sdde] Mode Sense: db 00 10 08 [ 39.302887] sd 0:0:47:0: [sdau] Write Protect is off [ 39.302888] sd 0:0:47:0: [sdau] Mode Sense: db 00 10 08 [ 39.302900] sd 0:0:169:0: [sdfk] Write Protect is off [ 39.302902] sd 0:0:169:0: [sdfk] Mode Sense: db 00 10 08 [ 39.302912] sd 0:0:156:0: [sdex] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302914] sd 0:0:156:0: [sdex] 4096-byte physical blocks [ 39.302938] sd 0:0:161:0: [sdfc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.302978] sd 0:0:187:0: [sdgb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302981] sd 0:0:187:0: [sdgb] 4096-byte physical blocks [ 39.302986] sd 0:0:65:0: [sdbl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.302987] sd 0:0:65:0: [sdbl] 4096-byte physical blocks [ 39.303062] sd 0:0:62:0: [sdbi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303076] sd 0:0:230:0: [sdhs] Write Protect is off [ 39.303078] sd 0:0:230:0: [sdhs] Mode Sense: db 00 10 08 [ 39.303082] sd 0:0:112:0: [sddg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303099] sd 0:0:107:0: [sddb] Write Protect is off [ 39.303101] sd 0:0:107:0: [sddb] Mode Sense: db 00 10 08 [ 39.303116] sd 0:0:195:0: [sdgj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303177] sd 0:0:226:0: [sdho] Write Protect is off [ 39.303179] sd 0:0:202:0: [sdgq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303180] sd 0:0:226:0: [sdho] Mode Sense: db 00 10 08 [ 39.303213] sd 0:0:74:0: [sdbu] Write Protect is off [ 39.303214] sd 0:0:74:0: [sdbu] Mode Sense: db 00 10 08 [ 39.303227] sd 0:0:86:0: [sdcg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303238] sd 0:0:82:0: [sdcc] Write Protect is off [ 39.303239] sd 0:0:82:0: [sdcc] Mode Sense: db 00 10 08 [ 39.303241] sd 0:0:114:0: [sddi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.303243] sd 0:0:106:0: [sdda] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303243] sd 0:0:114:0: [sddi] 4096-byte physical blocks [ 39.303356] sd 0:0:175:0: [sdfq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303386] sd 0:0:232:0: [sdhu] Write Protect is off [ 39.303388] sd 0:0:232:0: [sdhu] Mode Sense: db 00 10 08 [ 39.303397] sd 0:0:221:0: [sdhj] Write Protect is off [ 39.303399] sd 0:0:221:0: [sdhj] Mode Sense: db 00 10 08 [ 39.303416] sd 0:0:184:0: [sdfy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303427] sd 0:0:239:0: [sdib] Write Protect is off [ 39.303428] sd 0:0:239:0: [sdib] Mode Sense: db 00 10 08 [ 39.303451] sd 0:0:59:0: [sdbg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.303452] sd 0:0:59:0: [sdbg] 4096-byte physical blocks [ 39.303498] sd 0:0:185:0: [sdfz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303499] sd 0:0:209:0: [sdgx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303511] sd 0:0:218:0: [sdhg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303530] sd 0:0:236:0: [sdhy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.303532] sd 0:0:236:0: [sdhy] 4096-byte physical blocks [ 39.303537] sd 0:0:231:0: [sdht] Write Protect is off [ 39.303539] sd 0:0:231:0: [sdht] Mode Sense: db 00 10 08 [ 39.303588] sd 0:0:177:0: [sdfs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303651] sd 0:0:166:0: [sdfh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303670] sd 0:0:233:0: [sdhv] Write Protect is off [ 39.303672] sd 0:0:233:0: [sdhv] Mode Sense: db 00 10 08 [ 39.303672] sd 0:0:234:0: [sdhw] Write Protect is off [ 39.303673] sd 0:0:234:0: [sdhw] Mode Sense: db 00 10 08 [ 39.303682] sd 0:0:208:0: [sdgw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303684] sd 0:0:238:0: [sdia] Write Protect is off [ 39.303686] sd 0:0:238:0: [sdia] Mode Sense: db 00 10 08 [ 39.303736] sd 0:0:229:0: [sdhr] Write Protect is off [ 39.303737] sd 0:0:229:0: [sdhr] Mode Sense: db 00 10 08 [ 39.303782] sd 0:0:212:0: [sdha] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303791] sd 0:0:181:0: [sdfw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.303792] sd 0:0:181:0: [sdfw] 4096-byte physical blocks [ 39.303856] sd 0:0:196:0: [sdgk] Write Protect is off [ 39.303857] sd 0:0:196:0: [sdgk] Mode Sense: db 00 10 08 [ 39.303913] sd 0:0:102:0: [sdcw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.303915] sd 0:0:102:0: [sdcw] 4096-byte physical blocks [ 39.303922] sd 0:0:68:0: [sdbo] Write Protect is off [ 39.303924] sd 0:0:68:0: [sdbo] Mode Sense: db 00 10 08 [ 39.303937] sd 0:0:216:0: [sdhe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303984] sd 0:0:222:0: [sdhk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.303999] sd 0:0:153:0: [sdeu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304012] sd 0:0:83:0: [sdcd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304014] sd 0:0:83:0: [sdcd] 4096-byte physical blocks [ 39.304040] sd 0:0:69:0: [sdbp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304074] sd 0:0:240:0: [sdic] Write Protect is off [ 39.304076] sd 0:0:240:0: [sdic] Mode Sense: db 00 10 08 [ 39.304085] sd 0:0:66:0: [sdbm] Write Protect is off [ 39.304086] sd 0:0:66:0: [sdbm] Mode Sense: db 00 10 08 [ 39.304167] sd 0:0:204:0: [sdgs] Write Protect is off [ 39.304168] sd 0:0:204:0: [sdgs] Mode Sense: db 00 10 08 [ 39.304236] sd 0:0:71:0: [sdbr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304267] sd 0:0:224:0: [sdhm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304269] sd 0:0:224:0: [sdhm] 4096-byte physical blocks [ 39.304276] sd 0:0:84:0: [sdce] Write Protect is off [ 39.304278] sd 0:0:84:0: [sdce] Mode Sense: db 00 10 08 [ 39.304279] sd 0:0:63:0: [sdbj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304295] sd 0:0:223:0: [sdhl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304304] sd 0:0:72:0: [sdbs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304306] sd 0:0:72:0: [sdbs] 4096-byte physical blocks [ 39.304342] sd 0:0:67:0: [sdbn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304343] sd 0:0:85:0: [sdcf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304400] sd 0:0:26:0: [sdz] Write Protect is off [ 39.304401] sd 0:0:26:0: [sdz] Mode Sense: db 00 10 08 [ 39.304424] sd 0:0:60:0: [sdbh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304463] sd 0:0:105:0: [sdcz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304464] sd 0:0:105:0: [sdcz] 4096-byte physical blocks [ 39.304468] sd 0:0:70:0: [sdbq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304482] sd 0:0:190:0: [sdge] Write Protect is off [ 39.304484] sd 0:0:190:0: [sdge] Mode Sense: db 00 10 08 [ 39.304525] sd 0:0:205:0: [sdgt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304526] sd 0:0:243:0: [sdif] Write Protect is off [ 39.304527] sd 0:0:205:0: [sdgt] 4096-byte physical blocks [ 39.304528] sd 0:0:243:0: [sdif] Mode Sense: db 00 10 08 [ 39.304655] sd 0:0:194:0: [sdgi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304656] sd 0:0:194:0: [sdgi] 4096-byte physical blocks [ 39.304666] sd 0:0:79:0: [sdbz] Write Protect is off [ 39.304667] sd 0:0:79:0: [sdbz] Mode Sense: db 00 10 08 [ 39.304675] sd 0:0:191:0: [sdgf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304715] sd 0:0:163:0: [sdfe] Write Protect is off [ 39.304716] sd 0:0:163:0: [sdfe] Mode Sense: db 00 10 08 [ 39.304765] sd 0:0:188:0: [sdgc] Write Protect is off [ 39.304766] sd 0:0:188:0: [sdgc] Mode Sense: db 00 10 08 [ 39.304774] sd 0:0:228:0: [sdhq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304860] sd 0:0:227:0: [sdhp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304861] sd 0:0:206:0: [sdgu] Write Protect is off [ 39.304863] sd 0:0:227:0: [sdhp] 4096-byte physical blocks [ 39.304864] sd 0:0:235:0: [sdhx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304865] sd 0:0:206:0: [sdgu] Mode Sense: db 00 10 08 [ 39.304875] sd 0:0:193:0: [sdgh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.304960] sd 0:0:95:0: [sdcp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.304961] sd 0:0:95:0: [sdcp] 4096-byte physical blocks [ 39.304983] sd 0:0:189:0: [sdgd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305091] sd 0:0:92:0: [sdcm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305092] sd 0:0:92:0: [sdcm] 4096-byte physical blocks [ 39.305104] sd 0:0:182:0: [sdfx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305115] sd 0:0:159:0: [sdfa] Write Protect is off [ 39.305116] sd 0:0:159:0: [sdfa] Mode Sense: db 00 10 08 [ 39.305122] sd 0:0:207:0: [sdgv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305130] sd 0:0:103:0: [sdcx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305132] sd 0:0:103:0: [sdcx] 4096-byte physical blocks [ 39.305133] sd 0:0:241:0: [sdid] Write Protect is off [ 39.305134] sd 0:0:241:0: [sdid] Mode Sense: db 00 10 08 [ 39.305154] sd 0:0:197:0: [sdgl] Write Protect is off [ 39.305155] sd 0:0:197:0: [sdgl] Mode Sense: db 00 10 08 [ 39.305213] sd 0:0:230:0: [sdhs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305293] sd 0:0:148:0: [sdep] Write Protect is off [ 39.305294] sd 0:0:41:0: [sdao] Write Protect is off [ 39.305295] sd 0:0:148:0: [sdep] Mode Sense: db 00 10 08 [ 39.305296] sd 0:0:41:0: [sdao] Mode Sense: db 00 10 08 [ 39.305299] sd 0:0:158:0: [sdez] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305300] sd 0:0:158:0: [sdez] 4096-byte physical blocks [ 39.305302] sd 0:0:157:0: [sdey] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305304] sd 0:0:157:0: [sdey] 4096-byte physical blocks [ 39.305346] sd 0:0:88:0: [sdci] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305348] sd 0:0:88:0: [sdci] 4096-byte physical blocks [ 39.305368] sd 0:0:226:0: [sdho] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305390] sd 0:0:20:0: [sdt] Write Protect is off [ 39.305391] sd 0:0:20:0: [sdt] Mode Sense: db 00 10 08 [ 39.305433] sd 0:0:99:0: [sdct] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305450] sd 0:0:75:0: [sdbv] Write Protect is off [ 39.305451] sd 0:0:75:0: [sdbv] Mode Sense: db 00 10 08 [ 39.305460] sd 0:0:110:0: [sdde] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305468] sd 0:0:192:0: [sdgg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305493] sd 0:0:217:0: [sdhf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305495] sd 0:0:217:0: [sdhf] 4096-byte physical blocks [ 39.305509] sd 0:0:27:0: [sdaa] Write Protect is off [ 39.305511] sd 0:0:27:0: [sdaa] Mode Sense: db 00 10 08 [ 39.305516] sd 0:0:47:0: [sdau] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305541] sd 0:0:81:0: [sdcb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305543] sd 0:0:81:0: [sdcb] 4096-byte physical blocks [ 39.305586] sd 0:0:225:0: [sdhn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305588] sd 0:0:225:0: [sdhn] 4096-byte physical blocks [ 39.305596] sd 0:0:186:0: [sdga] Write Protect is off [ 39.305597] sd 0:0:186:0: [sdga] Mode Sense: db 00 10 08 [ 39.305609] sd 0:0:201:0: [sdgp] Write Protect is off [ 39.305611] sd 0:0:201:0: [sdgp] Mode Sense: db 00 10 08 [ 39.305612] sd 0:0:214:0: [sdhc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305614] sd 0:0:214:0: [sdhc] 4096-byte physical blocks [ 39.305647] sd 0:0:57:0: [sdbe] Write Protect is off [ 39.305648] sd 0:0:57:0: [sdbe] Mode Sense: db 00 10 08 [ 39.305671] sd 0:0:33:0: [sdag] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305672] sd 0:0:33:0: [sdag] 4096-byte physical blocks [ 39.305712] sd 0:0:237:0: [sdhz] Write Protect is off [ 39.305713] sd 0:0:237:0: [sdhz] Mode Sense: db 00 10 08 [ 39.305731] sd 0:0:9:0: [sdi] Write Protect is off [ 39.305732] sd 0:0:231:0: [sdht] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305734] sd 0:0:9:0: [sdi] Mode Sense: db 00 10 08 [ 39.305738] sd 0:0:210:0: [sdgy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305745] sd 0:0:210:0: [sdgy] 4096-byte physical blocks [ 39.305758] sd 0:0:239:0: [sdib] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305781] sd 0:0:160:0: [sdfb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305783] sd 0:0:160:0: [sdfb] 4096-byte physical blocks [ 39.305820] sd 0:0:22:0: [sdv] Write Protect is off [ 39.305822] sd 0:0:22:0: [sdv] Mode Sense: db 00 10 08 [ 39.305827] sd 0:0:203:0: [sdgr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305829] sd 0:0:203:0: [sdgr] 4096-byte physical blocks [ 39.305860] sd 0:0:7:0: [sdg] Write Protect is off [ 39.305862] sd 0:0:7:0: [sdg] Mode Sense: db 00 10 08 [ 39.305927] sd 0:0:169:0: [sdfk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.305967] sd 0:0:89:0: [sdcj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.305971] sd 0:0:89:0: [sdcj] 4096-byte physical blocks [ 39.306008] sd 0:0:91:0: [sdcl] Write Protect is off [ 39.306009] sd 0:0:91:0: [sdcl] Mode Sense: db 00 10 08 [ 39.306014] sd 0:0:56:0: [sdbd] Write Protect is off [ 39.306015] sd 0:0:56:0: [sdbd] Mode Sense: db 00 10 08 [ 39.306018] sd 0:0:97:0: [sdcr] Write Protect is off [ 39.306020] sd 0:0:97:0: [sdcr] Mode Sense: db 00 10 08 [ 39.306033] sd 0:0:107:0: [sddb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306034] sd 0:0:155:0: [sdew] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.306035] sd 0:0:155:0: [sdew] 4096-byte physical blocks [ 39.306040] sd 0:0:74:0: [sdbu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306044] sd 0:0:221:0: [sdhj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306069] sd 0:0:238:0: [sdia] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306104] sd 0:0:82:0: [sdcc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306109] sd 0:0:78:0: [sdby] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.306111] sd 0:0:78:0: [sdby] 4096-byte physical blocks [ 39.306136] sd 0:0:232:0: [sdhu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306138] sd 0:0:233:0: [sdhv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306142] sd 0:0:234:0: [sdhw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306160] sd 0:0:14:0: [sdn] Write Protect is off [ 39.306161] sd 0:0:14:0: [sdn] Mode Sense: db 00 10 08 [ 39.306174] sd 0:0:13:0: [sdm] Write Protect is off [ 39.306176] sd 0:0:13:0: [sdm] Mode Sense: db 00 10 08 [ 39.306209] sd 0:0:211:0: [sdgz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.306211] sd 0:0:211:0: [sdgz] 4096-byte physical blocks [ 39.306221] sd 0:0:64:0: [sdbk] Write Protect is off [ 39.306223] sd 0:0:64:0: [sdbk] Mode Sense: db 00 10 08 [ 39.306246] sd 0:0:17:0: [sdq] Write Protect is off [ 39.306248] sd 0:0:17:0: [sdq] Mode Sense: db 00 10 08 [ 39.306382] sd 0:0:25:0: [sdy] Write Protect is off [ 39.306383] sd 0:0:25:0: [sdy] Mode Sense: db 00 10 08 [ 39.306421] sd 0:0:115:0: [sddj] Write Protect is off [ 39.306423] sd 0:0:115:0: [sddj] Mode Sense: db 00 10 08 [ 39.306427] sd 0:0:200:0: [sdgo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.306429] sd 0:0:200:0: [sdgo] 4096-byte physical blocks [ 39.306431] sd 0:0:229:0: [sdhr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306497] sd 0:0:179:0: [sdfu] Write Protect is off [ 39.306499] sd 0:0:179:0: [sdfu] Mode Sense: db 00 10 08 [ 39.306532] sd 0:0:98:0: [sdcs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.306533] sd 0:0:98:0: [sdcs] 4096-byte physical blocks [ 39.306545] sd 0:0:240:0: [sdic] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306553] sd 0:0:213:0: [sdhb] Write Protect is off [ 39.306555] sd 0:0:213:0: [sdhb] Mode Sense: db 00 10 08 [ 39.306581] sd 0:0:11:0: [sdk] Write Protect is off [ 39.306583] sd 0:0:11:0: [sdk] Mode Sense: db 00 10 08 [ 39.306630] sd 0:0:50:0: [sdax] Write Protect is off [ 39.306631] sd 0:0:50:0: [sdax] Mode Sense: db 00 10 08 [ 39.306668] sd 0:0:219:0: [sdhh] Write Protect is off [ 39.306670] sd 0:0:219:0: [sdhh] Mode Sense: db 00 10 08 [ 39.306677] sd 0:0:196:0: [sdgk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306679] sd 0:0:204:0: [sdgs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306732] sd 0:0:178:0: [sdft] Write Protect is off [ 39.306734] sd 0:0:178:0: [sdft] Mode Sense: db 00 10 08 [ 39.306790] sd 0:0:68:0: [sdbo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.306856] sd 0:0:93:0: [sdcn] Write Protect is off [ 39.306858] sd 0:0:93:0: [sdcn] Mode Sense: db 00 10 08 [ 39.306901] sd 0:0:52:0: [sdaz] Write Protect is off [ 39.306902] sd 0:0:52:0: [sdaz] Mode Sense: db 00 10 08 [ 39.306945] sd 0:0:31:0: [sdae] Write Protect is off [ 39.306947] sd 0:0:31:0: [sdae] Mode Sense: db 00 10 08 [ 39.307008] sd 0:0:66:0: [sdbm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307026] sd 0:0:220:0: [sdhi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.307027] sd 0:0:220:0: [sdhi] 4096-byte physical blocks [ 39.307078] sd 0:0:18:0: [sdr] Write Protect is off [ 39.307080] sd 0:0:18:0: [sdr] Mode Sense: db 00 10 08 [ 39.307101] sd 0:0:120:0: [sddo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.307102] sd 0:0:163:0: [sdfe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307103] sd 0:0:120:0: [sddo] 4096-byte physical blocks [ 39.307110] sd 0:0:84:0: [sdce] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307182] sd 0:0:26:0: [sdz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307320] sd 0:0:172:0: [sdfn] Write Protect is off [ 39.307322] sd 0:0:172:0: [sdfn] Mode Sense: db 00 10 08 [ 39.307414] sd 0:0:190:0: [sdge] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307433] sd 0:0:173:0: [sdfo] Write Protect is off [ 39.307434] sd 0:0:173:0: [sdfo] Mode Sense: db 00 10 08 [ 39.307473] sd 0:0:32:0: [sdaf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.307476] sd 0:0:79:0: [sdbz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307478] sd 0:0:32:0: [sdaf] 4096-byte physical blocks [ 39.307548] sd 0:0:243:0: [sdif] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307578] sd 0:0:174:0: [sdfp] Write Protect is off [ 39.307580] sd 0:0:174:0: [sdfp] Mode Sense: db 00 10 08 [ 39.307620] sd 0:0:159:0: [sdfa] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307647] sd 0:0:215:0: [sdhd] Write Protect is off [ 39.307649] sd 0:0:215:0: [sdhd] Mode Sense: db 00 10 08 [ 39.307668] sd 0:0:206:0: [sdgu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307670] sd 0:0:242:0: [sdie] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.307671] sd 0:0:242:0: [sdie] 4096-byte physical blocks [ 39.307693] sd 0:0:188:0: [sdgc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307701] sd 0:0:241:0: [sdid] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307747] sd 0:0:148:0: [sdep] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307765] sd 0:0:187:0: [sdgb] Write Protect is off [ 39.307767] sd 0:0:187:0: [sdgb] Mode Sense: db 00 10 08 [ 39.307767] sd 0:0:41:0: [sdao] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307791] sd 0:0:20:0: [sdt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307809] sd 0:0:197:0: [sdgl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307848] sd 0:0:156:0: [sdex] Write Protect is off [ 39.307850] sd 0:0:156:0: [sdex] Mode Sense: db 00 10 08 [ 39.307874] sd 0:0:154:0: [sdev] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.307875] sd 0:0:154:0: [sdev] 4096-byte physical blocks [ 39.307955] sd 0:0:27:0: [sdaa] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.307986] sd 0:0:201:0: [sdgp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308059] sd 0:0:51:0: [sday] Write Protect is off [ 39.308060] sd 0:0:51:0: [sday] Mode Sense: db 00 10 08 [ 39.308085] sd 0:0:114:0: [sddi] Write Protect is off [ 39.308087] sd 0:0:114:0: [sddi] Mode Sense: db 00 10 08 [ 39.308141] sd 0:0:186:0: [sdga] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308155] sd 0:0:75:0: [sdbv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308236] sd 0:0:57:0: [sdbe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308248] sd 0:0:237:0: [sdhz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308286] sd 0:0:22:0: [sdv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308313] sd 0:0:7:0: [sdg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308344] sd 0:0:9:0: [sdi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308351] sd 0:0:65:0: [sdbl] Write Protect is off [ 39.308353] sd 0:0:65:0: [sdbl] Mode Sense: db 00 10 08 [ 39.308427] sd 0:0:56:0: [sdbd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308477] sd 0:0:91:0: [sdcl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308479] sd 0:0:14:0: [sdn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308485] sd 0:0:97:0: [sdcr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308545] sd 0:0:17:0: [sdq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308560] sd 0:0:13:0: [sdm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308640] sd 0:0:59:0: [sdbg] Write Protect is off [ 39.308642] sd 0:0:59:0: [sdbg] Mode Sense: db 00 10 08 [ 39.308653] sd 0:0:25:0: [sdy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308723] sd 0:0:64:0: [sdbk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308789] sd 0:0:179:0: [sdfu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308836] sd 0:0:11:0: [sdk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308909] sd 0:0:115:0: [sddj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.308957] sd 0:0:50:0: [sdax] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309061] sd 0:0:178:0: [sdft] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309119] sd 0:0:181:0: [sdfw] Write Protect is off [ 39.309121] sd 0:0:181:0: [sdfw] Mode Sense: db 00 10 08 [ 39.309140] sd 0:0:31:0: [sdae] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309214] sd 0:0:34:0: [sdah] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.309216] sd 0:0:34:0: [sdah] 4096-byte physical blocks [ 39.309235] sd 0:0:18:0: [sdr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309270] sd 0:0:236:0: [sdhy] Write Protect is off [ 39.309271] sd 0:0:236:0: [sdhy] Mode Sense: db 00 10 08 [ 39.309316] sd 0:0:52:0: [sdaz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309444] sd 0:0:83:0: [sdcd] Write Protect is off [ 39.309445] sd 0:0:83:0: [sdcd] Mode Sense: db 00 10 08 [ 39.309526] sd 0:0:93:0: [sdcn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309545] sd 0:0:72:0: [sdbs] Write Protect is off [ 39.309547] sd 0:0:72:0: [sdbs] Mode Sense: db 00 10 08 [ 39.309553] sd 0:0:102:0: [sdcw] Write Protect is off [ 39.309554] sd 0:0:102:0: [sdcw] Mode Sense: db 00 10 08 [ 39.309576] sd 0:0:219:0: [sdhh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309605] sd 0:0:213:0: [sdhb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309711] sd 0:0:105:0: [sdcz] Write Protect is off [ 39.309713] sd 0:0:105:0: [sdcz] Mode Sense: db 00 10 08 [ 39.309874] sd 0:0:172:0: [sdfn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309876] sd 0:0:173:0: [sdfo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.309938] sd 0:0:158:0: [sdez] Write Protect is off [ 39.309939] sd 0:0:158:0: [sdez] Mode Sense: db 00 10 08 [ 39.310018] sd 0:0:95:0: [sdcp] Write Protect is off [ 39.310020] sd 0:0:95:0: [sdcp] Mode Sense: db 00 10 08 [ 39.310031] sd 0:0:227:0: [sdhp] Write Protect is off [ 39.310032] sd 0:0:227:0: [sdhp] Mode Sense: db 00 10 08 [ 39.310047] sd 0:0:194:0: [sdgi] Write Protect is off [ 39.310048] sd 0:0:194:0: [sdgi] Mode Sense: db 00 10 08 [ 39.310078] sd 0:0:205:0: [sdgt] Write Protect is off [ 39.310079] sd 0:0:205:0: [sdgt] Mode Sense: db 00 10 08 [ 39.310086] sd 0:0:187:0: [sdgb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.310103] sd 0:0:224:0: [sdhm] Write Protect is off [ 39.310105] sd 0:0:224:0: [sdhm] Mode Sense: db 00 10 08 [ 39.310107] sd 0:0:157:0: [sdey] Write Protect is off [ 39.310108] sd 0:0:157:0: [sdey] Mode Sense: db 00 10 08 [ 39.310129] sd 0:0:174:0: [sdfp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.310131] sd 0:0:156:0: [sdex] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.310144] sd 0:0:215:0: [sdhd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.310259] sd 0:0:92:0: [sdcm] Write Protect is off [ 39.310261] sd 0:0:92:0: [sdcm] Mode Sense: db 00 10 08 [ 39.310415] sd 0:0:51:0: [sday] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.310483] sd 0:0:160:0: [sdfb] Write Protect is off [ 39.310485] sd 0:0:160:0: [sdfb] Mode Sense: db 00 10 08 [ 39.310501] sd 0:0:103:0: [sdcx] Write Protect is off [ 39.310502] sd 0:0:103:0: [sdcx] Mode Sense: db 00 10 08 [ 39.310596] sd 0:0:33:0: [sdag] Write Protect is off [ 39.310597] sd 0:0:33:0: [sdag] Mode Sense: db 00 10 08 [ 39.310664] sd 0:0:217:0: [sdhf] Write Protect is off [ 39.310667] sd 0:0:217:0: [sdhf] Mode Sense: db 00 10 08 [ 39.310800] sd 0:0:88:0: [sdci] Write Protect is off [ 39.310803] sd 0:0:88:0: [sdci] Mode Sense: db 00 10 08 [ 39.310806] sd 0:0:81:0: [sdcb] Write Protect is off [ 39.310807] sd 0:0:81:0: [sdcb] Mode Sense: db 00 10 08 [ 39.311145] sd 0:0:78:0: [sdby] Write Protect is off [ 39.311146] sd 0:0:78:0: [sdby] Mode Sense: db 00 10 08 [ 39.311157] sd 0:0:65:0: [sdbl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.311158] sd 0:0:89:0: [sdcj] Write Protect is off [ 39.311159] sd 0:0:89:0: [sdcj] Mode Sense: db 00 10 08 [ 39.311178] sd 0:0:114:0: [sddi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.311206] sd 0:0:59:0: [sdbg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.311276] sd 0:0:225:0: [sdhn] Write Protect is off [ 39.311277] sd 0:0:225:0: [sdhn] Mode Sense: db 00 10 08 [ 39.311278] sd 0:0:203:0: [sdgr] Write Protect is off [ 39.311279] sd 0:0:203:0: [sdgr] Mode Sense: db 00 10 08 [ 39.311398] sd 0:0:98:0: [sdcs] Write Protect is off [ 39.311400] sd 0:0:98:0: [sdcs] Mode Sense: db 00 10 08 [ 39.311421] sd 0:0:210:0: [sdgy] Write Protect is off [ 39.311423] sd 0:0:210:0: [sdgy] Mode Sense: db 00 10 08 [ 39.311677] sd 0:0:155:0: [sdew] Write Protect is off [ 39.311678] sd 0:0:155:0: [sdew] Mode Sense: db 00 10 08 [ 39.311906] sd 0:0:83:0: [sdcd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.311916] sd 0:0:72:0: [sdbs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.311942] sd 0:0:236:0: [sdhy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.311977] sd 0:0:102:0: [sdcw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312113] sd 0:0:211:0: [sdgz] Write Protect is off [ 39.312116] sd 0:0:211:0: [sdgz] Mode Sense: db 00 10 08 [ 39.312194] sd 0:0:181:0: [sdfw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312353] sd 0:0:32:0: [sdaf] Write Protect is off [ 39.312356] sd 0:0:32:0: [sdaf] Mode Sense: db 00 10 08 [ 39.312359] sd 0:0:120:0: [sddo] Write Protect is off [ 39.312495] sd 0:0:105:0: [sdcz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312519] sd 0:0:194:0: [sdgi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312520] sd 0:0:200:0: [sdgo] Write Protect is off [ 39.312522] sd 0:0:200:0: [sdgo] Mode Sense: db 00 10 08 [ 39.312539] sd 0:0:205:0: [sdgt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312634] sd 0:0:214:0: [sdhc] Write Protect is off [ 39.312636] sd 0:0:214:0: [sdhc] Mode Sense: db 00 10 08 [ 39.312649] sd 0:0:92:0: [sdcm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312724] sd 0:0:224:0: [sdhm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312753] sd 0:0:95:0: [sdcp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.312956] sd 0:0:120:0: [sddo] Mode Sense: db 00 10 08 [ 39.313002] sd 0:0:220:0: [sdhi] Write Protect is off [ 39.313004] sd 0:0:220:0: [sdhi] Mode Sense: db 00 10 08 [ 39.313006] sd 0:0:157:0: [sdey] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313014] sd 0:0:154:0: [sdev] Write Protect is off [ 39.313016] sd 0:0:154:0: [sdev] Mode Sense: db 00 10 08 [ 39.313048] sd 0:0:158:0: [sdez] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313069] sd 0:0:81:0: [sdcb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313087] sd 0:0:88:0: [sdci] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313090] sd 0:0:227:0: [sdhp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313200] sd 0:0:160:0: [sdfb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313523] sd 0:0:217:0: [sdhf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313552] sd 0:0:103:0: [sdcx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.313845] sd 0:0:89:0: [sdcj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314042] sd 0:0:203:0: [sdgr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314079] sd 0:0:33:0: [sdag] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314303] sd 0:0:210:0: [sdgy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314431] sd 0:0:78:0: [sdby] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314441] sd 0:0:225:0: [sdhn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314450] sd 0:0:242:0: [sdie] Write Protect is off [ 39.314452] sd 0:0:242:0: [sdie] Mode Sense: db 00 10 08 [ 39.314475] sd 0:0:211:0: [sdgz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314708] sd 0:0:214:0: [sdhc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.314940] sd 0:0:155:0: [sdew] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.315091] sd 0:0:200:0: [sdgo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.315371] sd 0:0:98:0: [sdcs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.315476] sd 0:0:34:0: [sdah] Write Protect is off [ 39.315481] sd 0:0:34:0: [sdah] Mode Sense: db 00 10 08 [ 39.315683] sd 0:0:220:0: [sdhi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.315790] sd 0:0:109:0: [sddd] Attached SCSI disk [ 39.315894] sd 0:0:32:0: [sdaf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.316496] sd 0:0:154:0: [sdev] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.316779] sd 0:0:137:0: [sdee] Attached SCSI disk [ 39.316828] sd 0:0:242:0: [sdie] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.316951] sd 0:0:35:0: [sdai] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.316954] sd 0:0:35:0: [sdai] 4096-byte physical blocks [ 39.317086] sd 0:0:117:0: [sddl] Attached SCSI disk [ 39.317455] sd 0:0:101:0: [sdcv] Attached SCSI disk [ 39.317636] sd 0:0:34:0: [sdah] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.318447] sd 0:0:100:0: [sdcu] Attached SCSI disk [ 39.318594] sd 0:0:116:0: [sddk] Attached SCSI disk [ 39.318849] sd 0:0:76:0: [sdbw] Attached SCSI disk [ 39.319899] sd 0:0:120:0: [sddo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.320630] sd 0:0:58:0: [sdbf] Attached SCSI disk [ 39.321028] sd 0:0:133:0: [sdea] Attached SCSI disk [ 39.321031] sd 0:0:129:0: [sddw] Attached SCSI disk [ 39.321881] sd 0:0:35:0: [sdai] Write Protect is off [ 39.321883] sd 0:0:35:0: [sdai] Mode Sense: db 00 10 08 [ 39.322407] sd 0:0:139:0: [sdeg] Attached SCSI disk [ 39.322886] sd 0:0:147:0: [sdeo] Attached SCSI disk [ 39.322970] sd 0:0:136:0: [sded] Attached SCSI disk [ 39.323462] sddb: [ 39.324420] sd 0:0:135:0: [sdec] Attached SCSI disk [ 39.324437] sd 0:0:35:0: [sdai] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.324465] sd 0:0:2:0: [sdb] Attached SCSI disk [ 39.324551] sd 0:0:54:0: [sdbb] Attached SCSI disk [ 39.324940] sdhr: [ 39.325360] sd 0:0:140:0: [sdeh] Attached SCSI disk [ 39.326037] sd 0:0:131:0: [sddy] Attached SCSI disk [ 39.326055] sd 0:0:126:0: [sddt] Attached SCSI disk [ 39.326094] sd 0:0:46:0: [sdat] Attached SCSI disk [ 39.326132] sd 0:0:3:0: [sdc] Attached SCSI disk [ 39.326252] sd 0:0:144:0: [sdel] Attached SCSI disk [ 39.326801] sd 0:0:132:0: [sddz] Attached SCSI disk [ 39.326994] sd 0:0:128:0: [sddv] Attached SCSI disk [ 39.327076] sd 0:0:130:0: [sddx] Attached SCSI disk [ 39.327168] sd 0:0:36:0: [sdaj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.327169] sd 0:0:36:0: [sdaj] 4096-byte physical blocks [ 39.327205] sd 0:0:43:0: [sdaq] Attached SCSI disk [ 39.327219] sd 0:0:124:0: [sddr] Attached SCSI disk [ 39.327347] sd 0:0:149:0: [sdeq] Attached SCSI disk [ 39.327435] sd 0:0:1:0: [sda] Attached SCSI disk [ 39.327610] sd 0:0:138:0: [sdef] Attached SCSI disk [ 39.327647] sd 0:0:30:0: [sdad] Attached SCSI disk [ 39.327711] sd 0:0:96:0: [sdcq] Attached SCSI disk [ 39.327837] sd 0:0:29:0: [sdac] Attached SCSI disk [ 39.327867] sd 0:0:104:0: [sdcy] Attached SCSI disk [ 39.327909] sd 0:0:123:0: [sddq] Attached SCSI disk [ 39.327910] sd 0:0:141:0: [sdei] Attached SCSI disk [ 39.327922] sd 0:0:73:0: [sdbt] Attached SCSI disk [ 39.328002] sd 0:0:21:0: [sdu] Attached SCSI disk [ 39.328118] sd 0:0:146:0: [sden] Attached SCSI disk [ 39.328209] sd 0:0:40:0: [sdan] Attached SCSI disk [ 39.328341] sd 0:0:28:0: [sdab] Attached SCSI disk [ 39.328351] sd 0:0:16:0: [sdp] Attached SCSI disk [ 39.328365] sd 0:0:37:0: [sdak] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.328367] sd 0:0:37:0: [sdak] 4096-byte physical blocks [ 39.328374] sd 0:0:127:0: [sddu] Attached SCSI disk [ 39.328430] sd 0:0:4:0: [sdd] Attached SCSI disk [ 39.328452] sd 0:0:8:0: [sdh] Attached SCSI disk [ 39.328463] sd 0:0:10:0: [sdj] Attached SCSI disk [ 39.328472] sd 0:0:38:0: [sdal] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.328474] sd 0:0:38:0: [sdal] 4096-byte physical blocks [ 39.328478] sd 0:0:12:0: [sdl] Attached SCSI disk [ 39.328510] sd 0:0:143:0: [sdek] Attached SCSI disk [ 39.328600] sd 0:0:125:0: [sdds] Attached SCSI disk [ 39.328708] sd 0:0:77:0: [sdbx] Attached SCSI disk [ 39.328812] sd 0:0:24:0: [sdx] Attached SCSI disk [ 39.328824] sd 0:0:134:0: [sdeb] Attached SCSI disk [ 39.328918] sd 0:0:23:0: [sdw] Attached SCSI disk [ 39.328990] sd 0:0:5:0: [sde] Attached SCSI disk [ 39.329094] sd 0:0:39:0: [sdam] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 39.329096] sd 0:0:39:0: [sdam] 4096-byte physical blocks [ 39.329352] sd 0:0:6:0: [sdf] Attached SCSI disk [ 39.329365] sd 0:0:19:0: [sds] Attached SCSI disk [ 39.329592] sd 0:0:164:0: [sdff] Attached SCSI disk [ 39.329764] sd 0:0:113:0: [sddh] Attached SCSI disk [ 39.329795] sd 0:0:42:0: [sdap] Attached SCSI disk [ 39.329972] sd 0:0:45:0: [sdas] Attached SCSI disk [ 39.329978] sd 0:0:151:0: [sdes] Attached SCSI disk [ 39.330087] sd 0:0:49:0: [sdaw] Attached SCSI disk [ 39.330244] sd 0:0:145:0: [sdem] Attached SCSI disk [ 39.330380] sd 0:0:48:0: [sdav] Attached SCSI disk [ 39.330421] sd 0:0:165:0: [sdfg] Attached SCSI disk [ 39.330428] sd 0:0:162:0: [sdfd] Attached SCSI disk [ 39.330487] sd 0:0:168:0: [sdfj] Attached SCSI disk [ 39.330771] sd 0:0:152:0: [sdet] Attached SCSI disk [ 39.330790] sd 0:0:180:0: [sdfv] Attached SCSI disk [ 39.331228] sd 0:0:111:0: [sddf] Attached SCSI disk [ 39.331325] sd 0:0:176:0: [sdfr] Attached SCSI disk [ 39.331328] sd 0:0:171:0: [sdfm] Attached SCSI disk [ 39.331340] sd 0:0:142:0: [sdej] Attached SCSI disk [ 39.331438] sd 0:0:36:0: [sdaj] Write Protect is off [ 39.331441] sd 0:0:36:0: [sdaj] Mode Sense: db 00 10 08 [ 39.331545] sd 0:0:53:0: [sdba] Attached SCSI disk [ 39.331624] sd 0:0:121:0: [sddp] Attached SCSI disk [ 39.331671] sd 0:0:161:0: [sdfc] Attached SCSI disk [ 39.331682] sd 0:0:80:0: [sdca] Attached SCSI disk [ 39.331751] sd 0:0:108:0: [sddc] Attached SCSI disk [ 39.331779] sd 0:0:167:0: [sdfi] Attached SCSI disk [ 39.331792] sd 0:0:87:0: [sdch] Attached SCSI disk [ 39.331822] sd 0:0:198:0: [sdgm] Attached SCSI disk [ 39.332028] sd 0:0:166:0: [sdfh] Attached SCSI disk [ 39.332046] sd 0:0:90:0: [sdck] Attached SCSI disk [ 39.332130] sd 0:0:184:0: [sdfy] Attached SCSI disk [ 39.332203] sd 0:0:38:0: [sdal] Write Protect is off [ 39.332205] sd 0:0:38:0: [sdal] Mode Sense: db 00 10 08 [ 39.332220] sd 0:0:118:0: [sddm] Attached SCSI disk [ 39.332296] sd 0:0:44:0: [sdar] Attached SCSI disk [ 39.332336] sd 0:0:222:0: [sdhk] Attached SCSI disk [ 39.332347] sd 0:0:94:0: [sdco] Attached SCSI disk [ 39.332363] sd 0:0:62:0: [sdbi] Attached SCSI disk [ 39.332372] sd 0:0:39:0: [sdam] Write Protect is off [ 39.332374] sd 0:0:39:0: [sdam] Mode Sense: db 00 10 08 [ 39.332454] sd 0:0:60:0: [sdbh] Attached SCSI disk [ 39.332473] sd 0:0:150:0: [sder] Attached SCSI disk [ 39.332704] sd 0:0:170:0: [sdfl] Attached SCSI disk [ 39.332754] sd 0:0:209:0: [sdgx] Attached SCSI disk [ 39.332844] sd 0:0:112:0: [sddg] Attached SCSI disk [ 39.332868] sd 0:0:37:0: [sdak] Write Protect is off [ 39.332869] sd 0:0:37:0: [sdak] Mode Sense: db 00 10 08 [ 39.332941] sd 0:0:63:0: [sdbj] Attached SCSI disk [ 39.333000] sd 0:0:119:0: [sddn] Attached SCSI disk [ 39.333029] sd 0:0:199:0: [sdgn] Attached SCSI disk [ 39.333086] sd 0:0:230:0: [sdhs] Attached SCSI disk [ 39.333094] sd 0:0:195:0: [sdgj] Attached SCSI disk [ 39.333104] sd 0:0:55:0: [sdbc] Attached SCSI disk [ 39.333144] sd 0:0:36:0: [sdaj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.333147] sd 0:0:69:0: [sdbp] Attached SCSI disk [ 39.333162] sd 0:0:239:0: [sdib] Attached SCSI disk [ 39.333190] sd 0:0:191:0: [sdgf] Attached SCSI disk [ 39.333216] sd 0:0:106:0: [sdda] Attached SCSI disk [ 39.333251] sd 0:0:223:0: [sdhl] Attached SCSI disk [ 39.333270] sd 0:0:202:0: [sdgq] Attached SCSI disk [ 39.333313] sd 0:0:175:0: [sdfq] Attached SCSI disk [ 39.333375] sd 0:0:208:0: [sdgw] Attached SCSI disk [ 39.333600] sd 0:0:231:0: [sdht] Attached SCSI disk [ 39.333633] sd 0:0:71:0: [sdbr] Attached SCSI disk [ 39.333649] sd 0:0:86:0: [sdcg] Attached SCSI disk [ 39.333650] sd 0:0:39:0: [sdam] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.333663] sd 0:0:226:0: [sdho] Attached SCSI disk [ 39.333711] sd 0:0:11:0: [sdk] Attached SCSI disk [ 39.333839] sd 0:0:212:0: [sdha] Attached SCSI disk [ 39.333852] sd 0:0:218:0: [sdhg] Attached SCSI disk [ 39.333899] sd 0:0:74:0: [sdbu] Attached SCSI disk [ 39.333948] sd 0:0:238:0: [sdia] Attached SCSI disk [ 39.334016] sd 0:0:177:0: [sdfs] Attached SCSI disk [ 39.334140] sd 0:0:169:0: [sdfk] Attached SCSI disk [ 39.334224] sd 0:0:38:0: [sdal] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.334253] sd 0:0:14:0: [sdn] Attached SCSI disk [ 39.334254] sd 0:0:159:0: [sdfa] Attached SCSI disk [ 39.334272] sd 0:0:193:0: [sdgh] Attached SCSI disk [ 39.334308] sd 0:0:216:0: [sdhe] Attached SCSI disk [ 39.334344] sd 0:0:22:0: [sdv] Attached SCSI disk [ 39.334352] sd 0:0:27:0: [sdaa] Attached SCSI disk [ 39.334358] sd 0:0:47:0: [sdau] Attached SCSI disk [ 39.334384] sd 0:0:85:0: [sdcf] Attached SCSI disk [ 39.334396] sd 0:0:185:0: [sdfz] Attached SCSI disk [ 39.334452] sd 0:0:13:0: [sdm] Attached SCSI disk [ 39.334530] sd 0:0:235:0: [sdhx] Attached SCSI disk [ 39.334598] sd 0:0:17:0: [sdq] Attached SCSI disk [ 39.334611] sd 0:0:25:0: [sdy] Attached SCSI disk [ 39.334779] sd 0:0:67:0: [sdbn] Attached SCSI disk [ 39.334857] sd 0:0:153:0: [sdeu] Attached SCSI disk [ 39.334871] sd 0:0:107:0: [sddb] Attached SCSI disk [ 39.334898] sd 0:0:82:0: [sdcc] Attached SCSI disk [ 39.334908] sd 0:0:196:0: [sdgk] Attached SCSI disk [ 39.334966] sd 0:0:20:0: [sdt] Attached SCSI disk [ 39.334972] sd 0:0:37:0: [sdak] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.335013] sd 0:0:232:0: [sdhu] Attached SCSI disk [ 39.335043] sd 0:0:7:0: [sdg] Attached SCSI disk [ 39.335095] sd 0:0:233:0: [sdhv] Attached SCSI disk [ 39.335125] sd 0:0:9:0: [sdi] Attached SCSI disk [ 39.335184] sd 0:0:110:0: [sdde] Attached SCSI disk [ 39.335227] sd 0:0:192:0: [sdgg] Attached SCSI disk [ 39.335270] sd 0:0:18:0: [sdr] Attached SCSI disk [ 39.335290] sd 0:0:99:0: [sdct] Attached SCSI disk [ 39.335470] sd 0:0:70:0: [sdbq] Attached SCSI disk [ 39.335570] sd 0:0:189:0: [sdgd] Attached SCSI disk [ 39.335668] sd 0:0:234:0: [sdhw] Attached SCSI disk [ 39.335680] sd 0:0:207:0: [sdgv] Attached SCSI disk [ 39.335703] sd 0:0:26:0: [sdz] Attached SCSI disk [ 39.335731] sd 0:0:204:0: [sdgs] Attached SCSI disk [ 39.335749] sd 0:0:229:0: [sdhr] Attached SCSI disk [ 39.335783] sd 0:0:84:0: [sdce] Attached SCSI disk [ 39.335794] sd 0:0:68:0: [sdbo] Attached SCSI disk [ 39.335858] sd 0:0:241:0: [sdid] Attached SCSI disk [ 39.335889] sd 0:0:243:0: [sdif] Attached SCSI disk [ 39.335919] sd 0:0:221:0: [sdhj] Attached SCSI disk [ 39.336007] sd 0:0:31:0: [sdae] Attached SCSI disk [ 39.336064] sd 0:0:79:0: [sdbz] Attached SCSI disk [ 39.336128] sd 0:0:190:0: [sdge] Attached SCSI disk [ 39.336140] sd 0:0:197:0: [sdgl] Attached SCSI disk [ 39.336166] sd 0:0:186:0: [sdga] Attached SCSI disk [ 39.336190] sd 0:0:56:0: [sdbd] Attached SCSI disk [ 39.336402] sd 0:0:201:0: [sdgp] Attached SCSI disk [ 39.336426] sd 0:0:148:0: [sdep] Attached SCSI disk [ 39.336462] sd 0:0:228:0: [sdhq] Attached SCSI disk [ 39.336526] sd 0:0:206:0: [sdgu] Attached SCSI disk [ 39.336581] sd 0:0:115:0: [sddj] Attached SCSI disk [ 39.336618] sd 0:0:188:0: [sdgc] Attached SCSI disk [ 39.336629] sd 0:0:240:0: [sdic] Attached SCSI disk [ 39.336652] sd 0:0:93:0: [sdcn] Attached SCSI disk [ 39.336731] sd 0:0:174:0: [sdfp] Attached SCSI disk [ 39.336745] sd 0:0:163:0: [sdfe] Attached SCSI disk [ 39.336807] sd 0:0:75:0: [sdbv] Attached SCSI disk [ 39.336883] sd 0:0:57:0: [sdbe] Attached SCSI disk [ 39.336926] sd 0:0:178:0: [sdft] Attached SCSI disk [ 39.337063] sd 0:0:41:0: [sdao] Attached SCSI disk [ 39.337081] sd 0:0:50:0: [sdax] Attached SCSI disk [ 39.337223] sd 0:0:179:0: [sdfu] Attached SCSI disk [ 39.337238] sd 0:0:64:0: [sdbk] Attached SCSI disk [ 39.337325] sd 0:0:237:0: [sdhz] Attached SCSI disk [ 39.337351] sd 0:0:66:0: [sdbm] Attached SCSI disk [ 39.337689] sd 0:0:236:0: [sdhy] Attached SCSI disk [ 39.337698] sd 0:0:51:0: [sday] Attached SCSI disk [ 39.337709] sd 0:0:156:0: [sdex] Attached SCSI disk [ 39.337781] sd 0:0:215:0: [sdhd] Attached SCSI disk [ 39.337905] sd 0:0:219:0: [sdhh] Attached SCSI disk [ 39.338009] sd 0:0:173:0: [sdfo] Attached SCSI disk [ 39.338024] sd 0:0:91:0: [sdcl] Attached SCSI disk [ 39.338187] sd 0:0:172:0: [sdfn] Attached SCSI disk [ 39.338345] sd 0:0:213:0: [sdhb] Attached SCSI disk [ 39.338405] sd 0:0:187:0: [sdgb] Attached SCSI disk [ 39.338424] sd 0:0:114:0: [sddi] Attached SCSI disk [ 39.338552] sd 0:0:194:0: [sdgi] Attached SCSI disk [ 39.338631] sd 0:0:97:0: [sdcr] Attached SCSI disk [ 39.338766] sd 0:0:83:0: [sdcd] Attached SCSI disk [ 39.338932] sd 0:0:224:0: [sdhm] Attached SCSI disk [ 39.338944] sd 0:0:59:0: [sdbg] Attached SCSI disk [ 39.339276] sd 0:0:102:0: [sdcw] Attached SCSI disk [ 39.339334] sd 0:0:158:0: [sdez] Attached SCSI disk [ 39.339411] sd 0:0:182:0: [sdfx] Attached SCSI disk [ 39.339522] sd 0:0:65:0: [sdbl] Attached SCSI disk [ 39.339707] sd 0:0:33:0: [sdag] Attached SCSI disk [ 39.339726] sd 0:0:72:0: [sdbs] Attached SCSI disk [ 39.339736] sd 0:0:227:0: [sdhp] Attached SCSI disk [ 39.339853] sd 0:0:160:0: [sdfb] Attached SCSI disk [ 39.339879] sd 0:0:52:0: [sdaz] Attached SCSI disk [ 39.339906] sd 0:0:217:0: [sdhf] Attached SCSI disk [ 39.340018] sd 0:0:203:0: [sdgr] Attached SCSI disk [ 39.340047] sd 0:0:105:0: [sdcz] Attached SCSI disk [ 39.340082] sd 0:0:181:0: [sdfw] Attached SCSI disk [ 39.340181] sd 0:0:157:0: [sdey] Attached SCSI disk [ 39.340222] sd 0:0:210:0: [sdgy] Attached SCSI disk [ 39.340489] sd 0:0:211:0: [sdgz] Attached SCSI disk [ 39.340523] sd 0:0:88:0: [sdci] Attached SCSI disk [ 39.340603] sd 0:0:95:0: [sdcp] Attached SCSI disk [ 39.340672] sd 0:0:78:0: [sdby] Attached SCSI disk [ 39.340756] sd 0:0:81:0: [sdcb] Attached SCSI disk [ 39.340799] sd 0:0:155:0: [sdew] Attached SCSI disk [ 39.340827] sd 0:0:89:0: [sdcj] Attached SCSI disk [ 39.340867] sd 0:0:32:0: [sdaf] Attached SCSI disk [ 39.340918] sd 0:0:103:0: [sdcx] Attached SCSI disk [ 39.340989] sd 0:0:200:0: [sdgo] Attached SCSI disk [ 39.341190] sd 0:0:220:0: [sdhi] Attached SCSI disk [ 39.341211] sd 0:0:34:0: [sdah] Attached SCSI disk [ 39.341233] sd 0:0:225:0: [sdhn] Attached SCSI disk [ 39.341523] sd 0:0:98:0: [sdcs] Attached SCSI disk [ 39.341595] sd 0:0:154:0: [sdev] Attached SCSI disk [ 39.341763] sd 0:0:92:0: [sdcm] Attached SCSI disk [ 39.342514] sd 0:0:214:0: [sdhc] Attached SCSI disk [ 39.342798] sd 0:0:242:0: [sdie] Attached SCSI disk [ 39.343924] sd 0:0:120:0: [sddo] Attached SCSI disk [ 39.344041] sd 0:0:205:0: [sdgt] Attached SCSI disk [ 39.345315] sd 0:0:35:0: [sdai] Attached SCSI disk [ 39.345991] sd 0:0:39:0: [sdam] Attached SCSI disk [ 39.347179] sd 0:0:37:0: [sdak] Attached SCSI disk [ 39.348808] sd 0:0:36:0: [sdaj] Attached SCSI disk [ 39.349435] sd 0:0:38:0: [sdal] Attached SCSI disk [ 39.913168] sd 0:0:15:0: [sdo] 4096-byte physical blocks [ 39.917850] sd 0:0:15:0: [sdo] Write Protect is off [ 39.917852] sd 0:0:15:0: [sdo] Mode Sense: db 00 10 08 [ 39.919397] sd 0:0:15:0: [sdo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 39.932075] sd 0:0:15:0: [sdo] Attached SCSI disk [ 47.187391] EXT4-fs (sdig2): mounted filesystem with ordered data mode. Opts: (null) [ 47.494692] systemd-journald[356]: Received SIGTERM from PID 1 (systemd). [ 47.523417] SELinux: Disabled at runtime. [ 47.528061] SELinux: Unregistering netfilter hooks [ 47.574961] type=1404 audit(1547777807.081:2): selinux=0 auid=4294967295 ses=4294967295 [ 47.601167] ip_tables: (C) 2000-2006 Netfilter Core Team [ 47.606973] systemd[1]: Inserted module 'ip_tables' [ 47.714834] EXT4-fs (sdig2): re-mounted. Opts: (null) [ 47.716186] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 47.733716] knem 1.1.3.90mlnx1: initialized [ 47.744668] systemd-journald[14679]: Received request to flush runtime journal from PID 1 [ 47.870844] ACPI Error: No handler for Region [SYSI] (ffff90cca9e82b40) [IPMI] (20130517/evregion-162) [ 47.884043] ACPI Error: Region IPMI (ID=7) has no handler (20130517/exfldio-305) [ 47.895652] ACPI Error: Method parse/execution failed [\_SB_.PMI0._GHL] (Node ffff90aca9e7b5a0), AE_NOT_EXIST (20130517/psparse-536) [ 47.927759] ipmi message handler version 39.2 [ 47.914586] ACPI Error: Method parse/execution failed [\_SB_.PMI0._PMC] (Node ffff90aca9e7b500), AE_NOT_EXIST (20130517/psparse-536) [ 47.939280] ACPI Exception: AE_NOT_EXIST, Evaluating _PMC (20130517/power_meter-753) [ 47.952484] ipmi device interface [ 47.953508] scsi 0:0:0:0: Attached scsi generic sg0 type 13 [ 47.953636] sd 0:0:1:0: Attached scsi generic sg1 type 0 [ 47.953773] sd 0:0:2:0: Attached scsi generic sg2 type 0 [ 47.954046] sd 0:0:3:0: Attached scsi generic sg3 type 0 [ 47.962576] sd 0:0:4:0: Attached scsi generic sg4 type 0 [ 47.962944] sd 0:0:5:0: Attached scsi generic sg5 type 0 [ 47.963285] sd 0:0:6:0: Attached scsi generic sg6 type 0 [ 47.963526] sd 0:0:7:0: Attached scsi generic sg7 type 0 [ 47.966486] sd 0:0:8:0: Attached scsi generic sg8 type 0 [ 47.966628] sd 0:0:9:0: Attached scsi generic sg9 type 0 [ 47.966934] sd 0:0:10:0: Attached scsi generic sg10 type 0 [ 47.968460] sd 0:0:11:0: Attached scsi generic sg11 type 0 [ 47.968896] sd 0:0:12:0: Attached scsi generic sg12 type 0 [ 48.038443] sd 0:0:13:0: Attached scsi generic sg13 type 0 [ 48.045514] sd 0:0:14:0: Attached scsi generic sg14 type 0 [ 48.053138] sd 0:0:15:0: Attached scsi generic sg15 type 0 [ 48.060799] sd 0:0:16:0: Attached scsi generic sg16 type 0 [ 48.061490] IPMI System Interface driver. [ 48.061591] ipmi_si ipmi_si.0: ipmi_platform: probing via SMBIOS [ 48.061594] ipmi_si: SMBIOS: io 0xca8 regsize 1 spacing 4 irq 10 [ 48.061595] ipmi_si: Adding SMBIOS-specified kcs state machine [ 48.061679] ipmi_si IPI0001:00: ipmi_platform: probing via ACPI [ 48.061707] ipmi_si IPI0001:00: [io 0x0ca8] regsize 1 spacing 4 irq 10 [ 48.061709] ipmi_si ipmi_si.0: Removing SMBIOS-specified kcs state machine in favor of ACPI [ 48.061710] ipmi_si: Adding ACPI-specified kcs state machine [ 48.062054] ipmi_si: Trying ACPI-specified kcs state machine at i/o address 0xca8, slave address 0x20, irq 10 [ 48.075014] ipmi_si IPI0001:00: The BMC does not support setting the recv irq bit, compensating, but the BMC needs to be fixed. [ 48.077294] ipmi_si IPI0001:00: Using irq 10 [ 48.140269] ipmi_si IPI0001:00: Found new BMC (man_id: 0x0002a2, prod_id: 0x0100, dev_id: 0x20) [ 48.167516] sd 0:0:17:0: Attached scsi generic sg17 type 0 [ 48.177360] sd 0:0:18:0: Attached scsi generic sg18 type 0 [ 48.187087] sd 0:0:19:0: Attached scsi generic sg19 type 0 [ 48.191061] device-mapper: uevent: version 1.0.3 [ 48.191476] device-mapper: ioctl: 4.37.1-ioctl (2018-04-03) initialised: dm-devel@redhat.com [ 48.209553] sd 0:0:20:0: Attached scsi generic sg20 type 0 [ 48.216285] sd 0:0:21:0: Attached scsi generic sg21 type 0 [ 48.223315] sd 0:0:22:0: Attached scsi generic sg22 type 0 [ 48.224585] ipmi_si IPI0001:00: IPMI kcs interface initialized [ 48.237128] sd 0:0:23:0: Attached scsi generic sg23 type 0 [ 48.246492] sd 0:0:24:0: Attached scsi generic sg24 type 0 [ 48.253741] sd 0:0:25:0: Attached scsi generic sg25 type 0 [ 48.260603] sd 0:0:26:0: Attached scsi generic sg26 type 0 [ 48.269400] sd 0:0:27:0: Attached scsi generic sg27 type 0 [ 48.276313] sd 0:0:28:0: Attached scsi generic sg28 type 0 [ 48.283385] sd 0:0:29:0: Attached scsi generic sg29 type 0 [ 48.291701] sd 0:0:30:0: Attached scsi generic sg30 type 0 [ 48.297753] sd 0:0:31:0: Attached scsi generic sg31 type 0 [ 48.310112] sd 0:0:32:0: Attached scsi generic sg32 type 0 [ 48.319263] sd 0:0:33:0: Attached scsi generic sg33 type 0 [ 48.326682] sd 0:0:34:0: Attached scsi generic sg34 type 0 [ 48.333349] sd 0:0:35:0: Attached scsi generic sg35 type 0 [ 48.340437] sd 0:0:36:0: Attached scsi generic sg36 type 0 [ 48.346941] sd 0:0:37:0: Attached scsi generic sg37 type 0 [ 48.354288] sd 0:0:38:0: Attached scsi generic sg38 type 0 [ 48.361165] sd 0:0:39:0: Attached scsi generic sg39 type 0 [ 48.368178] sd 0:0:40:0: Attached scsi generic sg40 type 0 [ 48.374643] sd 0:0:41:0: Attached scsi generic sg41 type 0 [ 48.382453] sd 0:0:42:0: Attached scsi generic sg42 type 0 [ 48.388456] sd 0:0:43:0: Attached scsi generic sg43 type 0 [ 48.395560] sd 0:0:44:0: Attached scsi generic sg44 type 0 [ 48.402179] sd 0:0:45:0: Attached scsi generic sg45 type 0 [ 48.409370] sd 0:0:46:0: Attached scsi generic sg46 type 0 [ 48.417264] sd 0:0:47:0: Attached scsi generic sg47 type 0 [ 48.424408] sd 0:0:48:0: Attached scsi generic sg48 type 0 [ 48.430788] sd 0:0:49:0: Attached scsi generic sg49 type 0 [ 48.438790] sd 0:0:50:0: Attached scsi generic sg50 type 0 [ 48.447011] sd 0:0:51:0: Attached scsi generic sg51 type 0 [ 48.453124] sd 0:0:52:0: Attached scsi generic sg52 type 0 [ 48.458966] sd 0:0:53:0: Attached scsi generic sg53 type 0 [ 48.466022] sd 0:0:54:0: Attached scsi generic sg54 type 0 [ 48.472293] sd 0:0:55:0: Attached scsi generic sg55 type 0 [ 48.479715] sd 0:0:56:0: Attached scsi generic sg56 type 0 [ 48.485584] sd 0:0:57:0: Attached scsi generic sg57 type 0 [ 48.495181] sd 0:0:58:0: Attached scsi generic sg58 type 0 [ 48.501021] sd 0:0:59:0: Attached scsi generic sg59 type 0 [ 48.506810] sd 0:0:60:0: Attached scsi generic sg60 type 0 [ 48.512805] scsi 0:0:61:0: Attached scsi generic sg61 type 13 [ 48.519310] sd 0:0:62:0: Attached scsi generic sg62 type 0 [ 48.525191] sd 0:0:63:0: Attached scsi generic sg63 type 0 [ 48.531151] sd 0:0:64:0: Attached scsi generic sg64 type 0 [ 48.537052] sd 0:0:65:0: Attached scsi generic sg65 type 0 [ 48.542827] sd 0:0:66:0: Attached scsi generic sg66 type 0 [ 48.548658] sd 0:0:67:0: Attached scsi generic sg67 type 0 [ 48.554353] sd 0:0:68:0: Attached scsi generic sg68 type 0 [ 48.560276] sd 0:0:69:0: Attached scsi generic sg69 type 0 [ 48.565967] sd 0:0:70:0: Attached scsi generic sg70 type 0 [ 48.572001] sd 0:0:71:0: Attached scsi generic sg71 type 0 [ 48.577557] sd 0:0:72:0: Attached scsi generic sg72 type 0 [ 48.583225] sd 0:0:73:0: Attached scsi generic sg73 type 0 [ 48.588927] sd 0:0:74:0: Attached scsi generic sg74 type 0 [ 48.594607] sd 0:0:75:0: Attached scsi generic sg75 type 0 [ 48.600253] sd 0:0:76:0: Attached scsi generic sg76 type 0 [ 48.605815] sd 0:0:77:0: Attached scsi generic sg77 type 0 [ 48.611373] sd 0:0:78:0: Attached scsi generic sg78 type 0 [ 48.616940] sd 0:0:79:0: Attached scsi generic sg79 type 0 [ 48.623824] sd 0:0:80:0: Attached scsi generic sg80 type 0 [ 48.630687] sd 0:0:81:0: Attached scsi generic sg81 type 0 [ 48.637563] sd 0:0:82:0: Attached scsi generic sg82 type 0 [ 48.644438] sd 0:0:83:0: Attached scsi generic sg83 type 0 [ 48.651325] sd 0:0:84:0: Attached scsi generic sg84 type 0 [ 48.658195] sd 0:0:85:0: Attached scsi generic sg85 type 0 [ 48.665059] sd 0:0:86:0: Attached scsi generic sg86 type 0 [ 48.671242] sd 0:0:87:0: Attached scsi generic sg87 type 0 [ 48.678107] sd 0:0:88:0: Attached scsi generic sg88 type 0 [ 48.685006] sd 0:0:89:0: Attached scsi generic sg89 type 0 [ 48.691852] sd 0:0:90:0: Attached scsi generic sg90 type 0 [ 48.698722] sd 0:0:91:0: Attached scsi generic sg91 type 0 [ 48.704827] sd 0:0:92:0: Attached scsi generic sg92 type 0 [ 48.710370] sd 0:0:93:0: Attached scsi generic sg93 type 0 [ 48.715896] sd 0:0:94:0: Attached scsi generic sg94 type 0 [ 48.721451] sd 0:0:95:0: Attached scsi generic sg95 type 0 [ 48.726990] sd 0:0:96:0: Attached scsi generic sg96 type 0 [ 48.732521] sd 0:0:97:0: Attached scsi generic sg97 type 0 [ 48.738055] sd 0:0:98:0: Attached scsi generic sg98 type 0 [ 48.743597] sd 0:0:99:0: Attached scsi generic sg99 type 0 [ 48.749134] sd 0:0:100:0: Attached scsi generic sg100 type 0 [ 48.754837] sd 0:0:101:0: Attached scsi generic sg101 type 0 [ 48.760561] sd 0:0:102:0: Attached scsi generic sg102 type 0 [ 48.766261] sd 0:0:103:0: Attached scsi generic sg103 type 0 [ 48.771966] sd 0:0:104:0: Attached scsi generic sg104 type 0 [ 48.777681] sd 0:0:105:0: Attached scsi generic sg105 type 0 [ 48.783394] sd 0:0:106:0: Attached scsi generic sg106 type 0 [ 48.789098] sd 0:0:107:0: Attached scsi generic sg107 type 0 [ 48.794819] sd 0:0:108:0: Attached scsi generic sg108 type 0 [ 48.800527] sd 0:0:109:0: Attached scsi generic sg109 type 0 [ 48.806250] sd 0:0:110:0: Attached scsi generic sg110 type 0 [ 48.811954] sd 0:0:111:0: Attached scsi generic sg111 type 0 [ 48.817661] sd 0:0:112:0: Attached scsi generic sg112 type 0 [ 48.823383] sd 0:0:113:0: Attached scsi generic sg113 type 0 [ 48.829089] sd 0:0:114:0: Attached scsi generic sg114 type 0 [ 48.834792] sd 0:0:115:0: Attached scsi generic sg115 type 0 [ 48.840513] sd 0:0:116:0: Attached scsi generic sg116 type 0 [ 48.846409] sd 0:0:117:0: Attached scsi generic sg117 type 0 [ 48.852120] sd 0:0:118:0: Attached scsi generic sg118 type 0 [ 48.857845] sd 0:0:119:0: Attached scsi generic sg119 type 0 [ 48.863558] sd 0:0:120:0: Attached scsi generic sg120 type 0 [ 48.869267] sd 0:0:121:0: Attached scsi generic sg121 type 0 [ 48.874996] scsi 0:0:122:0: Attached scsi generic sg122 type 13 [ 48.880963] sd 0:0:123:0: Attached scsi generic sg123 type 0 [ 48.886696] sd 0:0:124:0: Attached scsi generic sg124 type 0 [ 48.892405] sd 0:0:125:0: Attached scsi generic sg125 type 0 [ 48.898116] sd 0:0:126:0: Attached scsi generic sg126 type 0 [ 48.903844] sd 0:0:127:0: Attached scsi generic sg127 type 0 [ 48.909562] sd 0:0:128:0: Attached scsi generic sg128 type 0 [ 48.915270] sd 0:0:129:0: Attached scsi generic sg129 type 0 [ 48.921001] sd 0:0:130:0: Attached scsi generic sg130 type 0 [ 48.926710] sd 0:0:131:0: Attached scsi generic sg131 type 0 [ 48.932423] sd 0:0:132:0: Attached scsi generic sg132 type 0 [ 48.938146] sd 0:0:133:0: Attached scsi generic sg133 type 0 [ 48.943852] sd 0:0:134:0: Attached scsi generic sg134 type 0 [ 48.949564] sd 0:0:135:0: Attached scsi generic sg135 type 0 [ 48.955287] sd 0:0:136:0: Attached scsi generic sg136 type 0 [ 48.960994] sd 0:0:137:0: Attached scsi generic sg137 type 0 [ 48.966705] sd 0:0:138:0: Attached scsi generic sg138 type 0 [ 48.972431] sd 0:0:139:0: Attached scsi generic sg139 type 0 [ 48.978135] sd 0:0:140:0: Attached scsi generic sg140 type 0 [ 48.983856] sd 0:0:141:0: Attached scsi generic sg141 type 0 [ 48.989575] sd 0:0:142:0: Attached scsi generic sg142 type 0 [ 48.995280] sd 0:0:143:0: Attached scsi generic sg143 type 0 [ 49.000975] sd 0:0:144:0: Attached scsi generic sg144 type 0 [ 49.006707] sd 0:0:145:0: Attached scsi generic sg145 type 0 [ 49.012408] sd 0:0:146:0: Attached scsi generic sg146 type 0 [ 49.018118] sd 0:0:147:0: Attached scsi generic sg147 type 0 [ 49.023824] sd 0:0:148:0: Attached scsi generic sg148 type 0 [ 49.029531] sd 0:0:149:0: Attached scsi generic sg149 type 0 [ 49.035239] sd 0:0:150:0: Attached scsi generic sg150 type 0 [ 49.040955] sd 0:0:151:0: Attached scsi generic sg151 type 0 [ 49.046663] sd 0:0:152:0: Attached scsi generic sg152 type 0 [ 49.052383] sd 0:0:153:0: Attached scsi generic sg153 type 0 [ 49.058084] sd 0:0:154:0: Attached scsi generic sg154 type 0 [ 49.063788] sd 0:0:155:0: Attached scsi generic sg155 type 0 [ 49.069508] sd 0:0:156:0: Attached scsi generic sg156 type 0 [ 49.075216] sd 0:0:157:0: Attached scsi generic sg157 type 0 [ 49.080934] sd 0:0:158:0: Attached scsi generic sg158 type 0 [ 49.086636] sd 0:0:159:0: Attached scsi generic sg159 type 0 [ 49.092351] sd 0:0:160:0: Attached scsi generic sg160 type 0 [ 49.098058] sd 0:0:161:0: Attached scsi generic sg161 type 0 [ 49.103757] sd 0:0:162:0: Attached scsi generic sg162 type 0 [ 49.109481] sd 0:0:163:0: Attached scsi generic sg163 type 0 [ 49.115182] sd 0:0:164:0: Attached scsi generic sg164 type 0 [ 49.120890] sd 0:0:165:0: Attached scsi generic sg165 type 0 [ 49.126598] sd 0:0:166:0: Attached scsi generic sg166 type 0 [ 49.132324] sd 0:0:167:0: Attached scsi generic sg167 type 0 [ 49.138041] sd 0:0:168:0: Attached scsi generic sg168 type 0 [ 49.143758] sd 0:0:169:0: Attached scsi generic sg169 type 0 [ 49.149469] sd 0:0:170:0: Attached scsi generic sg170 type 0 [ 49.155192] sd 0:0:171:0: Attached scsi generic sg171 type 0 [ 49.160895] sd 0:0:172:0: Attached scsi generic sg172 type 0 [ 49.166601] sd 0:0:173:0: Attached scsi generic sg173 type 0 [ 49.172314] sd 0:0:174:0: Attached scsi generic sg174 type 0 [ 49.178041] sd 0:0:175:0: Attached scsi generic sg175 type 0 [ 49.183756] sd 0:0:176:0: Attached scsi generic sg176 type 0 [ 49.189470] sd 0:0:177:0: Attached scsi generic sg177 type 0 [ 49.195188] sd 0:0:178:0: Attached scsi generic sg178 type 0 [ 49.200895] sd 0:0:179:0: Attached scsi generic sg179 type 0 [ 49.206605] sd 0:0:180:0: Attached scsi generic sg180 type 0 [ 49.212334] sd 0:0:181:0: Attached scsi generic sg181 type 0 [ 49.218045] sd 0:0:182:0: Attached scsi generic sg182 type 0 [ 49.223757] scsi 0:0:183:0: Attached scsi generic sg183 type 13 [ 49.229746] sd 0:0:184:0: Attached scsi generic sg184 type 0 [ 49.235459] sd 0:0:185:0: Attached scsi generic sg185 type 0 [ 49.241180] sd 0:0:186:0: Attached scsi generic sg186 type 0 [ 49.246889] sd 0:0:187:0: Attached scsi generic sg187 type 0 [ 49.252601] sd 0:0:188:0: Attached scsi generic sg188 type 0 [ 49.258323] sd 0:0:189:0: Attached scsi generic sg189 type 0 [ 49.264032] sd 0:0:190:0: Attached scsi generic sg190 type 0 [ 49.269734] sd 0:0:191:0: Attached scsi generic sg191 type 0 [ 49.275454] sd 0:0:192:0: Attached scsi generic sg192 type 0 [ 49.281153] sd 0:0:193:0: Attached scsi generic sg193 type 0 [ 49.286855] sd 0:0:194:0: Attached scsi generic sg194 type 0 [ 49.292562] sd 0:0:195:0: Attached scsi generic sg195 type 0 [ 49.298277] sd 0:0:196:0: Attached scsi generic sg196 type 0 [ 49.303981] sd 0:0:197:0: Attached scsi generic sg197 type 0 [ 49.309687] sd 0:0:198:0: Attached scsi generic sg198 type 0 [ 49.315404] sd 0:0:199:0: Attached scsi generic sg199 type 0 [ 49.321109] sd 0:0:200:0: Attached scsi generic sg200 type 0 [ 49.326811] sd 0:0:201:0: Attached scsi generic sg201 type 0 [ 49.332517] sd 0:0:202:0: Attached scsi generic sg202 type 0 [ 49.338231] sd 0:0:203:0: Attached scsi generic sg203 type 0 [ 49.343939] sd 0:0:204:0: Attached scsi generic sg204 type 0 [ 49.349637] sd 0:0:205:0: Attached scsi generic sg205 type 0 [ 49.355358] sd 0:0:206:0: Attached scsi generic sg206 type 0 [ 49.361058] sd 0:0:207:0: Attached scsi generic sg207 type 0 [ 49.366768] sd 0:0:208:0: Attached scsi generic sg208 type 0 [ 49.372485] sd 0:0:209:0: Attached scsi generic sg209 type 0 [ 49.378194] sd 0:0:210:0: Attached scsi generic sg210 type 0 [ 49.383898] sd 0:0:211:0: Attached scsi generic sg211 type 0 [ 49.389622] sd 0:0:212:0: Attached scsi generic sg212 type 0 [ 49.395337] sd 0:0:213:0: Attached scsi generic sg213 type 0 [ 49.401048] sd 0:0:214:0: Attached scsi generic sg214 type 0 [ 49.406769] sd 0:0:215:0: Attached scsi generic sg215 type 0 [ 49.412480] sd 0:0:216:0: Attached scsi generic sg216 type 0 [ 49.418192] sd 0:0:217:0: Attached scsi generic sg217 type 0 [ 49.423912] sd 0:0:218:0: Attached scsi generic sg218 type 0 [ 49.429626] sd 0:0:219:0: Attached scsi generic sg219 type 0 [ 49.435336] sd 0:0:220:0: Attached scsi generic sg220 type 0 [ 49.441064] sd 0:0:221:0: Attached scsi generic sg221 type 0 [ 49.446772] sd 0:0:222:0: Attached scsi generic sg222 type 0 [ 49.452481] sd 0:0:223:0: Attached scsi generic sg223 type 0 [ 49.458206] sd 0:0:224:0: Attached scsi generic sg224 type 0 [ 49.463919] sd 0:0:225:0: Attached scsi generic sg225 type 0 [ 49.469630] sd 0:0:226:0: Attached scsi generic sg226 type 0 [ 49.475354] sd 0:0:227:0: Attached scsi generic sg227 type 0 [ 49.481061] sd 0:0:228:0: Attached scsi generic sg228 type 0 [ 49.486769] sd 0:0:229:0: Attached scsi generic sg229 type 0 [ 49.492492] sd 0:0:230:0: Attached scsi generic sg230 type 0 [ 49.498200] sd 0:0:231:0: Attached scsi generic sg231 type 0 [ 49.503906] sd 0:0:232:0: Attached scsi generic sg232 type 0 [ 49.509626] sd 0:0:233:0: Attached scsi generic sg233 type 0 [ 49.515353] sd 0:0:234:0: Attached scsi generic sg234 type 0 [ 49.521058] sd 0:0:235:0: Attached scsi generic sg235 type 0 [ 49.526768] sd 0:0:236:0: Attached scsi generic sg236 type 0 [ 49.532473] sd 0:0:237:0: Attached scsi generic sg237 type 0 [ 49.538195] sd 0:0:238:0: Attached scsi generic sg238 type 0 [ 49.543900] sd 0:0:239:0: Attached scsi generic sg239 type 0 [ 49.549606] sd 0:0:240:0: Attached scsi generic sg240 type 0 [ 49.555327] sd 0:0:241:0: Attached scsi generic sg241 type 0 [ 49.561039] sd 0:0:242:0: Attached scsi generic sg242 type 0 [ 49.566746] sd 0:0:243:0: Attached scsi generic sg243 type 0 [ 49.572460] scsi 0:0:244:0: Attached scsi generic sg244 type 13 [ 49.578437] sd 1:2:0:0: Attached scsi generic sg245 type 0 [ 49.584156] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.592741] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.601147] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.609599] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.618017] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.626434] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.634862] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.643301] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.651725] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.660151] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.668564] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.676992] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.685408] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.693848] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.702267] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.710680] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.719090] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.727487] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.735885] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.744293] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.752687] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.761083] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.769475] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.777863] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.786257] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.794655] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.803052] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.811454] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.819846] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.828237] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.836640] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.845042] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.853428] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.861825] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.870221] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.878619] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.887019] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.895413] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.903801] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.912190] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.920584] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.928978] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.937372] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.945768] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.954164] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.962555] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.970941] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.979331] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.987721] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 49.996137] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.004532] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.012933] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.021324] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.029718] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.038114] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.046504] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.054899] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.063329] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.071733] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.080124] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.088520] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.096915] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.105309] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.113696] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.122089] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.130485] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.138872] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.147263] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.155651] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.164051] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.172441] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.180829] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.189224] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.197620] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.206011] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.214411] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.222807] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.231205] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.239590] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.247989] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.256380] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.264776] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.273167] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.281556] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.289948] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.298338] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.306726] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.315132] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.323533] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.331933] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.340333] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.348743] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.357222] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.365614] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.374015] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.382411] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.390808] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.399214] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.407612] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.416020] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.424422] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.432817] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.441213] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.449626] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.458018] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.466423] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.474815] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.483221] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.491620] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.500030] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.508439] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.516833] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.525230] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.533626] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.542015] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.550410] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.558808] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.567218] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.575610] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.583999] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.592388] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.600787] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.609183] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.617584] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.625980] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.634382] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.642778] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.651174] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.659566] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.667969] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.676361] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.684762] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.693158] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.701550] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.709950] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.718352] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.726743] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.735143] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.743541] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.751941] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.760338] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.768745] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.777140] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.785540] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.793936] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.802341] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.810734] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.819137] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.827531] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.835932] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.844327] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.852724] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.861111] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.869512] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.877901] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.886300] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.894689] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.903090] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.911487] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.919878] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.928278] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.936683] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.945084] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.953482] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.961878] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.970275] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.978675] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.987077] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 50.995474] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.003869] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.012267] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.020666] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.029074] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.037484] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.045875] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.054276] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.062676] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.071078] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.079474] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.087872] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.096263] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.104664] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.113052] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.121452] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.129848] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.138247] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.146635] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.155038] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.163438] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.171833] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.180230] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.188626] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.197017] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.205410] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.213809] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.222212] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.230606] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.239034] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.247438] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.255837] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.264257] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.272702] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.281122] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.289550] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.297953] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.306361] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.314761] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.323161] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.331556] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.339999] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.348419] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.356834] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.365240] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.373672] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.382145] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.390584] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.399108] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.407534] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.415956] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.424394] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.432844] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.441268] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.449676] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.458104] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.466516] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.474941] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.483344] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.491777] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.500207] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.508659] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.517080] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.525527] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.533974] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.542417] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.550845] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.559285] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.567707] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.576155] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.584595] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.593055] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.601477] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.609919] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.618357] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.626797] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.635230] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.643650] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.652101] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.660528] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.668969] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.677403] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.685819] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.694234] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.702670] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.711122] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.719566] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.728027] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.736467] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.744932] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.753382] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.761848] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.770309] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.778783] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.787257] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.795725] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.804250] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.812779] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.821241] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.829699] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.838227] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.846722] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.855191] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.863657] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.872141] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.880627] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.889096] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.897594] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.906052] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.914538] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.923006] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.931497] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.939966] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.948448] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.956924] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.965428] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.973897] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982378] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990839] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999319] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.007783] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.016269] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.024735] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.033225] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.041783] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.050344] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.058885] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.067458] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.076017] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.084603] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.093181] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.101761] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.110254] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.545765] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.555880] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.565968] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.576063] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.586543] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.596678] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.606813] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.617210] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.627306] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.638042] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.648171] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.658284] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.669174] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.679745] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.689984] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.700190] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.710349] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.720503] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.730758] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.741633] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.752526] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.762977] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.773488] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.784081] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.794735] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.805007] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.815512] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.825664] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.835780] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.845997] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.856141] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.866275] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.876442] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.886555] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.896734] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.906875] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.916967] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.927132] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.937217] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.947393] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.957531] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.967658] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.977752] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.987834] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.997921] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.007997] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.018140] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.028243] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.038398] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.048525] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.058669] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.068807] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.078931] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.089066] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.099192] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.109323] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.119439] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.129953] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.140066] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.150216] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.225013] device-mapper: multipath service-time: version 0.3.0 loaded [ 53.592345] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.604375] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.616452] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.628336] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.640263] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.652118] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.664073] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.675894] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.687878] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.699696] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.712178] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.724041] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.735952] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.747807] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.759709] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.771574] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.783471] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.795306] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.807172] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.818984] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.830826] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.842674] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.854604] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.866495] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.878353] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.890191] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.902169] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.914086] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.926439] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.938602] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.950567] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.962487] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.974470] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.986423] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.998396] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.010635] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.022585] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.034440] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.046390] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.058311] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.070279] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.082269] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.094172] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.106017] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.117843] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.129630] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.141458] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.153240] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.165006] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.176778] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.188523] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.200284] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.212021] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.223810] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.235643] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.247579] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.259468] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.271271] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.283166] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.295503] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.727718] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.741250] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.754812] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.768410] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.782032] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.796045] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.809704] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.823285] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.836846] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.850323] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.863900] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.878218] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.891751] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.905232] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.918757] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.932230] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.946037] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.959612] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.973196] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.986769] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.000310] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.013807] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.027725] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.041330] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.054927] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.068592] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.082281] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.095945] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.109570] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.123210] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.136863] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.150483] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.164156] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.177915] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.191593] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.205979] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.220049] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.233686] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.247355] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.261186] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.274910] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.288597] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.302212] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.316278] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.331075] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.345079] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.359169] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.372836] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.387109] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.400810] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.414801] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.428520] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.442077] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.455625] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.469177] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.482716] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.496602] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.510161] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.523797] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.537833] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 55.988299] ses 0:0:0:0: Attached Enclosure device [ 55.993193] ses 0:0:61:0: Attached Enclosure device [ 55.998138] ses 0:0:122:0: Attached Enclosure device [ 56.003191] ses 0:0:183:0: Attached Enclosure device [ 56.008233] ses 0:0:244:0: Attached Enclosure device [ 56.355016] piix4_smbus 0000:00:14.0: SMBus Host Controller at 0xb00, revision 0 [ 56.362476] piix4_smbus 0000:00:14.0: Using register 0x2e for SMBus port selection [ 56.370924] ccp 0000:02:00.2: 3 command queues available [ 56.376356] ccp 0000:02:00.2: irq 238 for MSI/MSI-X [ 56.376372] ccp 0000:02:00.2: irq 239 for MSI/MSI-X [ 56.376435] ccp 0000:02:00.2: Queue 2 can access 4 LSB regions [ 56.382292] ccp 0000:02:00.2: Queue 3 can access 4 LSB regions [ 56.388152] ccp 0000:02:00.2: Queue 4 can access 4 LSB regions [ 56.394006] ccp 0000:02:00.2: Queue 0 gets LSB 4 [ 56.398645] ccp 0000:02:00.2: Queue 1 gets LSB 5 [ 56.403291] ccp 0000:02:00.2: Queue 2 gets LSB 6 [ 56.410031] ccp 0000:02:00.2: enabled [ 56.414004] ccp 0000:03:00.1: 5 command queues available [ 56.419403] ccp 0000:03:00.1: irq 241 for MSI/MSI-X [ 56.419469] ccp 0000:03:00.1: Queue 0 can access 7 LSB regions [ 56.425327] ccp 0000:03:00.1: Queue 1 can access 7 LSB regions [ 56.431170] ccp 0000:03:00.1: Queue 2 can access 7 LSB regions [ 56.437009] ccp 0000:03:00.1: Queue 3 can access 7 LSB regions [ 56.442851] ccp 0000:03:00.1: Queue 4 can access 7 LSB regions [ 56.448691] ccp 0000:03:00.1: Queue 0 gets LSB 1 [ 56.453318] ccp 0000:03:00.1: Queue 1 gets LSB 2 [ 56.457945] ccp 0000:03:00.1: Queue 2 gets LSB 3 [ 56.462563] ccp 0000:03:00.1: Queue 3 gets LSB 4 [ 56.467188] ccp 0000:03:00.1: Queue 4 gets LSB 5 [ 56.472439] ccp 0000:03:00.1: enabled [ 56.476375] ccp 0000:41:00.2: 3 command queues available [ 56.481801] ccp 0000:41:00.2: irq 243 for MSI/MSI-X [ 56.481818] ccp 0000:41:00.2: irq 244 for MSI/MSI-X [ 56.481899] ccp 0000:41:00.2: Queue 2 can access 4 LSB regions [ 56.487751] ccp 0000:41:00.2: Queue 3 can access 4 LSB regions [ 56.493605] ccp 0000:41:00.2: Queue 4 can access 4 LSB regions [ 56.499458] ccp 0000:41:00.2: Queue 0 gets LSB 4 [ 56.504093] ccp 0000:41:00.2: Queue 1 gets LSB 5 [ 56.508730] ccp 0000:41:00.2: Queue 2 gets LSB 6 [ 56.514085] ccp 0000:41:00.2: enabled [ 56.518226] ccp 0000:42:00.1: 5 command queues available [ 56.523813] ccp 0000:42:00.1: irq 246 for MSI/MSI-X [ 56.523872] ccp 0000:42:00.1: Queue 0 can access 7 LSB regions [ 56.529717] ccp 0000:42:00.1: Queue 1 can access 7 LSB regions [ 56.535577] ccp 0000:42:00.1: Queue 2 can access 7 LSB regions [ 56.541419] ccp 0000:42:00.1: Queue 3 can access 7 LSB regions [ 56.547258] ccp 0000:42:00.1: Queue 4 can access 7 LSB regions [ 56.553098] ccp 0000:42:00.1: Queue 0 gets LSB 1 [ 56.557715] ccp 0000:42:00.1: Queue 1 gets LSB 2 [ 56.562335] ccp 0000:42:00.1: Queue 2 gets LSB 3 [ 56.566955] ccp 0000:42:00.1: Queue 3 gets LSB 4 [ 56.571574] ccp 0000:42:00.1: Queue 4 gets LSB 5 [ 56.577365] ccp 0000:42:00.1: enabled [ 56.581320] ccp 0000:85:00.2: 3 command queues available [ 56.586759] ccp 0000:85:00.2: irq 248 for MSI/MSI-X [ 56.586773] ccp 0000:85:00.2: irq 249 for MSI/MSI-X [ 56.586838] ccp 0000:85:00.2: Queue 2 can access 4 LSB regions [ 56.592693] ccp 0000:85:00.2: Queue 3 can access 4 LSB regions [ 56.598547] ccp 0000:85:00.2: Queue 4 can access 4 LSB regions [ 56.604402] ccp 0000:85:00.2: Queue 0 gets LSB 4 [ 56.609043] ccp 0000:85:00.2: Queue 1 gets LSB 5 [ 56.613685] ccp 0000:85:00.2: Queue 2 gets LSB 6 [ 56.618736] ccp 0000:85:00.2: enabled [ 56.622673] ccp 0000:86:00.1: 5 command queues available [ 56.628112] ccp 0000:86:00.1: irq 251 for MSI/MSI-X [ 56.628161] ccp 0000:86:00.1: Queue 0 can access 7 LSB regions [ 56.634013] ccp 0000:86:00.1: Queue 1 can access 7 LSB regions [ 56.639869] ccp 0000:86:00.1: Queue 2 can access 7 LSB regions [ 56.645719] ccp 0000:86:00.1: Queue 3 can access 7 LSB regions [ 56.651573] ccp 0000:86:00.1: Queue 4 can access 7 LSB regions [ 56.657428] ccp 0000:86:00.1: Queue 0 gets LSB 1 [ 56.662064] ccp 0000:86:00.1: Queue 1 gets LSB 2 [ 56.666700] ccp 0000:86:00.1: Queue 2 gets LSB 3 [ 56.671338] ccp 0000:86:00.1: Queue 3 gets LSB 4 [ 56.675974] ccp 0000:86:00.1: Queue 4 gets LSB 5 [ 56.683266] ccp 0000:86:00.1: enabled [ 56.687200] ccp 0000:c2:00.2: 3 command queues available [ 56.692676] ccp 0000:c2:00.2: irq 253 for MSI/MSI-X [ 56.693423] ccp 0000:c2:00.2: irq 254 for MSI/MSI-X [ 56.693497] ccp 0000:c2:00.2: Queue 2 can access 4 LSB regions [ 56.693629] input: PC Speaker as /devices/platform/pcspkr/input/input2 [ 56.705895] ccp 0000:c2:00.2: Queue 3 can access 4 LSB regions [ 56.705900] ccp 0000:c2:00.2: Queue 4 can access 4 LSB regions [ 56.705902] ccp 0000:c2:00.2: Queue 0 gets LSB 4 [ 56.705903] ccp 0000:c2:00.2: Queue 1 gets LSB 5 [ 56.705904] ccp 0000:c2:00.2: Queue 2 gets LSB 6 [ 56.707693] ccp 0000:c2:00.2: enabled [ 56.707952] ccp 0000:c3:00.1: 5 command queues available [ 56.708021] ccp 0000:c3:00.1: irq 256 for MSI/MSI-X [ 56.708057] ccp 0000:c3:00.1: Queue 0 can access 7 LSB regions [ 56.708059] ccp 0000:c3:00.1: Queue 1 can access 7 LSB regions [ 56.708062] ccp 0000:c3:00.1: Queue 2 can access 7 LSB regions [ 56.708064] ccp 0000:c3:00.1: Queue 3 can access 7 LSB regions [ 56.708066] ccp 0000:c3:00.1: Queue 4 can access 7 LSB regions [ 56.708067] ccp 0000:c3:00.1: Queue 0 gets LSB 1 [ 56.708069] ccp 0000:c3:00.1: Queue 1 gets LSB 2 [ 56.708070] ccp 0000:c3:00.1: Queue 2 gets LSB 3 [ 56.708071] ccp 0000:c3:00.1: Queue 3 gets LSB 4 [ 56.708072] ccp 0000:c3:00.1: Queue 4 gets LSB 5 [ 56.708860] ccp 0000:c3:00.1: enabled [ 56.844106] cryptd: max_cpu_qlen set to 1000 [ 56.863299] AVX2 version of gcm_enc/dec engaged. [ 56.867928] AES CTR mode by8 optimization enabled [ 56.876313] alg: No test for __gcm-aes-aesni (__driver-gcm-aes-aesni) [ 56.882911] alg: No test for __generic-gcm-aes-aesni (__driver-generic-gcm-aes-aesni) [ 56.953522] kvm: Nested Paging enabled [ 56.963912] MCE: In-kernel MCE decoding enabled. [ 56.972433] AMD64 EDAC driver v3.4.0 [ 56.976039] EDAC amd64: DRAM ECC enabled. [ 56.980060] EDAC amd64: F17h detected (node 0). [ 56.984646] EDAC MC: UMC0 chip selects: [ 56.984648] EDAC amd64: MC: 0: 0MB 1: 0MB [ 56.989358] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 56.994072] EDAC amd64: MC: 4: 0MB 5: 0MB [ 56.998792] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.003534] EDAC MC: UMC1 chip selects: [ 57.003539] EDAC amd64: MC: 0: 0MB 1: 0MB [ 57.008265] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 57.012976] EDAC amd64: MC: 4: 0MB 5: 0MB [ 57.017701] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.022419] EDAC amd64: using x8 syndromes. [ 57.026613] EDAC amd64: MCT channel count: 2 [ 57.036453] EDAC MC0: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:18.3 [ 57.043865] EDAC amd64: DRAM ECC enabled. [ 57.048015] EDAC amd64: F17h detected (node 1). [ 57.052662] EDAC MC: UMC0 chip selects: [ 57.052665] EDAC amd64: MC: 0: 0MB 1: 0MB [ 57.057387] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 57.062107] EDAC amd64: MC: 4: 0MB 5: 0MB [ 57.066828] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.071561] EDAC MC: UMC1 chip selects: [ 57.071564] EDAC amd64: MC: 0: 0MB 1: 0MB [ 57.076306] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 57.081018] EDAC amd64: MC: 4: 0MB 5: 0MB [ 57.085736] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.090450] EDAC amd64: using x8 syndromes. [ 57.094648] EDAC amd64: MCT channel count: 2 [ 57.102183] EDAC MC1: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:19.3 [ 57.109602] EDAC amd64: DRAM ECC enabled. [ 57.113633] EDAC amd64: F17h detected (node 2). [ 57.118250] EDAC MC: UMC0 chip selects: [ 57.118254] EDAC amd64: MC: 0: 0MB 1: 0MB [ 57.122976] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 57.127696] EDAC amd64: MC: 4: 0MB 5: 0MB [ 57.132405] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.137130] EDAC MC: UMC1 chip selects: [ 57.137132] EDAC amd64: MC: 0: 0MB 1: 0MB [ 57.141845] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 57.146563] EDAC amd64: MC: 4: 0MB 5: 0MB [ 57.151276] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.155988] EDAC amd64: using x8 syndromes. [ 57.160181] EDAC amd64: MCT channel count: 2 [ 57.170493] EDAC MC2: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1a.3 [ 57.177897] EDAC amd64: DRAM ECC enabled. [ 57.181917] EDAC amd64: F17h detected (node 3). [ 57.186509] EDAC MC: UMC0 chip selects: [ 57.186513] EDAC amd64: MC: 0: 0MB 1: 0MB [ 57.191228] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 57.195944] EDAC amd64: MC: 4: 0MB 5: 0MB [ 57.200658] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.205375] EDAC MC: UMC1 chip selects: [ 57.205379] EDAC amd64: MC: 0: 0MB 1: 0MB [ 57.210097] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 57.214811] EDAC amd64: MC: 4: 0MB 5: 0MB [ 57.219526] EDAC amd64: MC: 6: 0MB 7: 0MB [ 57.224240] EDAC amd64: using x8 syndromes. [ 57.228433] EDAC amd64: MCT channel count: 2 [ 57.235191] EDAC MC3: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1b.3 [ 57.242735] EDAC PCI0: Giving out device to module 'amd64_edac' controller 'EDAC PCI controller': DEV '0000:00:18.0' (POLLED) [ 57.985344] dcdbas dcdbas: Dell Systems Management Base Driver (version 5.6.0-3.3) [ 100.920647] Adding 4194300k swap on /dev/sdig3. Priority:-2 extents:1 across:4194300k FS [ 100.963144] type=1305 audit(1547777860.467:3): audit_pid=71601 old=0 auid=4294967295 ses=4294967295 res=1 [ 100.984503] RPC: Registered named UNIX socket transport module. [ 100.990760] RPC: Registered udp transport module. [ 100.996851] RPC: Registered tcp transport module. [ 101.002944] RPC: Registered tcp NFSv4.1 backchannel transport module. [ 101.303094] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.381820] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.436549] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.564489] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.613896] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.697481] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.711426] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.727721] mlx5_core 0000:84:00.0: slow_pci_heuristic:5202:(pid 71893): Max link speed = 100000, PCI BW = 126016 [ 101.738079] mlx5_core 0000:84:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(64) RxCqeCmprss(0) [ 101.746269] mlx5_core 0000:84:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(64) RxCqeCmprss(0) [ 101.884355] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.897171] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.946865] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 101.995220] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 102.290140] tg3 0000:81:00.0: irq 257 for MSI/MSI-X [ 102.290153] tg3 0000:81:00.0: irq 258 for MSI/MSI-X [ 102.290163] tg3 0000:81:00.0: irq 259 for MSI/MSI-X [ 102.290173] tg3 0000:81:00.0: irq 260 for MSI/MSI-X [ 102.290184] tg3 0000:81:00.0: irq 261 for MSI/MSI-X [ 102.416303] IPv6: ADDRCONF(NETDEV_UP): em1: link is not ready [ 105.851295] tg3 0000:81:00.0 em1: Link is up at 1000 Mbps, full duplex [ 105.857827] tg3 0000:81:00.0 em1: Flow control is on for TX and on for RX [ 105.864619] tg3 0000:81:00.0 em1: EEE is enabled [ 105.869253] IPv6: ADDRCONF(NETDEV_CHANGE): em1: link becomes ready [ 106.653648] IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready [ 106.766637] IPv6: ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready [ 110.900165] FS-Cache: Loaded [ 110.929306] FS-Cache: Netfs 'nfs' registered for caching [ 110.938338] Key type dns_resolver registered [ 110.966610] NFS: Registering the id_resolver key type [ 110.971989] Key type id_resolver registered [ 110.977563] Key type id_legacy registered [ 3567.116678] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [ 3567.124477] alg: No test for adler32 (adler32-zlib) [ 3567.927303] Lustre: Lustre: Build Version: 2.12.0 [ 3568.051138] LNet: Using FastReg for registration [ 3568.067969] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [ 3572.373685] md: md11 stopped. [ 3572.396436] async_tx: api initialized (async) [ 3572.402794] xor: automatically using best checksumming function: [ 3572.418175] avx : 9640.000 MB/sec [ 3572.451181] raid6: sse2x1 gen() 6078 MB/s [ 3572.472178] raid6: sse2x2 gen() 11328 MB/s [ 3572.493179] raid6: sse2x4 gen() 12917 MB/s [ 3572.514181] raid6: avx2x1 gen() 14273 MB/s [ 3572.535180] raid6: avx2x2 gen() 18832 MB/s [ 3572.556179] raid6: avx2x4 gen() 18773 MB/s [ 3572.560451] raid6: using algorithm avx2x2 gen() (18832 MB/s) [ 3572.566113] raid6: using avx2x2 recovery algorithm [ 3572.586805] md/raid:md11: device dm-50 operational as raid disk 0 [ 3572.592904] md/raid:md11: device dm-46 operational as raid disk 9 [ 3572.599007] md/raid:md11: device dm-30 operational as raid disk 8 [ 3572.605109] md/raid:md11: device dm-112 operational as raid disk 7 [ 3572.611297] md/raid:md11: device dm-114 operational as raid disk 6 [ 3572.617482] md/raid:md11: device dm-80 operational as raid disk 5 [ 3572.623602] md/raid:md11: device dm-48 operational as raid disk 4 [ 3572.629701] md/raid:md11: device dm-40 operational as raid disk 3 [ 3572.635797] md/raid:md11: device dm-104 operational as raid disk 2 [ 3572.641982] md/raid:md11: device dm-95 operational as raid disk 1 [ 3572.649137] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3572.703977] md11: detected capacity change from 0 to 64011422924800 [ 3572.760924] md: md1 stopped. [ 3572.784059] md/raid:md1: device dm-26 operational as raid disk 0 [ 3572.790084] md/raid:md1: device dm-9 operational as raid disk 9 [ 3572.796020] md/raid:md1: device dm-16 operational as raid disk 8 [ 3572.802041] md/raid:md1: device dm-59 operational as raid disk 7 [ 3572.808060] md/raid:md1: device dm-23 operational as raid disk 6 [ 3572.814084] md/raid:md1: device dm-60 operational as raid disk 5 [ 3572.820104] md/raid:md1: device dm-34 operational as raid disk 4 [ 3572.826121] md/raid:md1: device dm-68 operational as raid disk 3 [ 3572.832142] md/raid:md1: device dm-43 operational as raid disk 2 [ 3572.838158] md/raid:md1: device dm-57 operational as raid disk 1 [ 3572.844947] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3573.090550] md1: detected capacity change from 0 to 64011422924800 [ 3573.118123] md: md3 stopped. [ 3573.182319] md/raid:md3: device dm-2 operational as raid disk 0 [ 3573.188250] md/raid:md3: device dm-0 operational as raid disk 9 [ 3573.194198] md/raid:md3: device dm-65 operational as raid disk 8 [ 3573.200211] md/raid:md3: device dm-24 operational as raid disk 7 [ 3573.206240] md/raid:md3: device dm-14 operational as raid disk 6 [ 3573.212250] md/raid:md3: device dm-118 operational as raid disk 5 [ 3573.218361] md/raid:md3: device dm-64 operational as raid disk 4 [ 3573.224375] md/raid:md3: device dm-75 operational as raid disk 3 [ 3573.230461] md/raid:md3: device dm-31 operational as raid disk 2 [ 3573.236508] md/raid:md3: device dm-15 operational as raid disk 1 [ 3573.243554] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3573.286430] md3: detected capacity change from 0 to 64011422924800 [ 3573.304362] md: md5 stopped. [ 3573.372660] LDISKFS-fs warning (device md11): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3573.390342] md/raid:md5: device dm-84 operational as raid disk 0 [ 3573.396366] md/raid:md5: device dm-8 operational as raid disk 9 [ 3573.402293] md/raid:md5: device dm-3 operational as raid disk 8 [ 3573.408218] md/raid:md5: device dm-22 operational as raid disk 7 [ 3573.414231] md/raid:md5: device dm-21 operational as raid disk 6 [ 3573.420244] md/raid:md5: device dm-67 operational as raid disk 5 [ 3573.426254] md/raid:md5: device dm-36 operational as raid disk 4 [ 3573.432266] md/raid:md5: device dm-10 operational as raid disk 3 [ 3573.438277] md/raid:md5: device dm-76 operational as raid disk 2 [ 3573.444292] md/raid:md5: device dm-56 operational as raid disk 1 [ 3573.451336] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3573.491547] md5: detected capacity change from 0 to 64011422924800 [ 3573.509658] md: md7 stopped. [ 3573.532882] md/raid:md7: device dm-92 operational as raid disk 0 [ 3573.538910] md/raid:md7: device dm-38 operational as raid disk 9 [ 3573.544938] md/raid:md7: device dm-49 operational as raid disk 8 [ 3573.550965] md/raid:md7: device dm-97 operational as raid disk 7 [ 3573.556989] md/raid:md7: device dm-111 operational as raid disk 6 [ 3573.563096] md/raid:md7: device dm-96 operational as raid disk 5 [ 3573.569119] md/raid:md7: device dm-101 operational as raid disk 4 [ 3573.575227] md/raid:md7: device dm-91 operational as raid disk 3 [ 3573.581251] md/raid:md7: device dm-86 operational as raid disk 2 [ 3573.587273] md/raid:md7: device dm-109 operational as raid disk 1 [ 3573.594593] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3573.624751] md7: detected capacity change from 0 to 64011422924800 [ 3573.637757] md: md9 stopped. [ 3573.659232] md/raid:md9: device dm-47 operational as raid disk 0 [ 3573.665262] md/raid:md9: device dm-42 operational as raid disk 9 [ 3573.671351] md/raid:md9: device dm-39 operational as raid disk 8 [ 3573.677388] md/raid:md9: device dm-28 operational as raid disk 7 [ 3573.683486] md/raid:md9: device dm-116 operational as raid disk 6 [ 3573.689615] md/raid:md9: device dm-71 operational as raid disk 5 [ 3573.695648] md/raid:md9: device dm-110 operational as raid disk 4 [ 3573.701761] md/raid:md9: device dm-94 operational as raid disk 3 [ 3573.707807] md/raid:md9: device dm-33 operational as raid disk 2 [ 3573.713891] md/raid:md9: device dm-53 operational as raid disk 1 [ 3573.720902] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3573.751456] md9: detected capacity change from 0 to 64011422924800 [ 3573.866065] LDISKFS-fs warning (device md1): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3574.127163] LDISKFS-fs warning (device md3): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3574.289386] LDISKFS-fs warning (device md5): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3574.513109] LDISKFS-fs warning (device md7): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3574.703559] LDISKFS-fs warning (device md9): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3615.448530] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 3615.943529] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 3616.214548] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 3616.288427] LDISKFS-fs (md11): recovery complete [ 3616.314313] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3616.368549] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 3616.602950] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 3616.835578] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 3616.841339] LDISKFS-fs (md1): recovery complete [ 3616.858021] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3617.069217] LDISKFS-fs (md3): recovery complete [ 3617.091658] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3617.331279] LDISKFS-fs (md5): recovery complete [ 3617.361112] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3617.500592] LDISKFS-fs (md7): recovery complete [ 3617.524926] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3617.861959] LDISKFS-fs (md9): recovery complete [ 3617.882144] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3620.051414] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [ 3622.051471] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [ 3624.051515] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 5 seconds [ 3626.051565] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 7 seconds [ 3626.061731] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [ 3626.073819] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [ 3626.085090] Lustre: fir-OST000b: in recovery but waiting for the first client to connect [ 3630.051666] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 11 seconds [ 3630.061918] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [ 3631.053929] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [ 3631.062660] Lustre: fir-OST0001: in recovery but waiting for the first client to connect [ 3635.284717] Lustre: fir-OST0001: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3635.285580] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3635.286582] Lustre: fir-OST000b: Connection restored to (at 10.8.9.6@o2ib6) [ 3635.318002] Lustre: Skipped 1 previous similar message [ 3636.054079] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [ 3636.065768] Lustre: fir-OST0003: in recovery but waiting for the first client to connect [ 3639.051896] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [ 3639.062154] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [ 3641.054240] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [ 3641.063028] Lustre: fir-OST0005: in recovery but waiting for the first client to connect [ 3642.352315] Lustre: fir-OST0003: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3642.352378] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.6.4@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3642.352379] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.8.6.4@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3642.352380] LustreError: Skipped 8 previous similar messages [ 3642.352381] LustreError: Skipped 8 previous similar messages [ 3642.352718] Lustre: fir-OST0001: Connection restored to (at 10.8.6.4@o2ib6) [ 3642.352720] Lustre: Skipped 1 previous similar message [ 3642.419157] Lustre: Skipped 1 previous similar message [ 3644.130924] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [ 3644.137981] Lustre: Skipped 4 previous similar messages [ 3646.054309] Lustre: fir-OST0007: Imperative Recovery not enabled, recovery window 300-900 [ 3646.063120] Lustre: fir-OST0007: in recovery but waiting for the first client to connect [ 3648.651183] Lustre: fir-OST0007: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3648.660366] Lustre: fir-OST0007: Connection restored to (at 10.8.6.4@o2ib6) [ 3651.750857] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3651.750858] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3651.750861] LustreError: Skipped 6 previous similar messages [ 3651.790779] LustreError: Skipped 3 previous similar messages [ 3655.684245] Lustre: fir-OST0009: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3655.693441] Lustre: fir-OST0009: Connection restored to (at 10.8.6.4@o2ib6) [ 3663.873978] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.101.59@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3663.891349] LustreError: Skipped 4 previous similar messages [ 3668.874335] Lustre: fir-OST0001: Connection restored to (at 10.9.101.59@o2ib4) [ 3668.881661] Lustre: Skipped 13 previous similar messages [ 3680.052945] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [ 3680.063114] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 11 previous similar messages [ 3685.286068] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3685.303174] LustreError: Skipped 11 previous similar messages [ 3706.752258] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3706.769391] LustreError: Skipped 5 previous similar messages [ 3774.561493] Lustre: fir-OST000b: Connection restored to (at 10.0.10.51@o2ib7) [ 3774.568718] Lustre: Skipped 11 previous similar messages [ 3774.751691] Lustre: fir-OST0007: Recovery over after 2:06, of 7 clients 7 recovered and 0 were evicted. [ 3774.761089] Lustre: Skipped 3 previous similar messages [ 3817.584403] Lustre: fir-OST0005: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [ 3817.593625] Lustre: Skipped 7 previous similar messages [ 3879.315664] Lustre: fir-OST0007: deleting orphan objects from 0x0:34748 to 0x0:34817 [ 3879.325753] Lustre: fir-OST0005: deleting orphan objects from 0x0:34748 to 0x0:34817 [ 3879.327284] Lustre: fir-OST000b: deleting orphan objects from 0x0:34748 to 0x0:34817 [ 3879.333594] Lustre: fir-OST0001: deleting orphan objects from 0x0:34748 to 0x0:34817 [ 3879.333612] Lustre: fir-OST0009: deleting orphan objects from 0x0:34748 to 0x0:34817 [ 3879.341367] Lustre: fir-OST0003: deleting orphan objects from 0x0:34747 to 0x0:34817 [ 4072.280077] Lustre: fir-OST0001: Connection restored to 30ed9acf-7f48-0f0d-1126-afaebb670d13 (at 10.0.10.3@o2ib7) [ 4072.290337] Lustre: Skipped 9 previous similar messages [ 4196.384163] Lustre: 6925:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547781949/real 1547781949] req@ffff90ab1e488000 x1622966352751216/t0(0) o39->fir-MDT0002-lwp-OST0003@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781955 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [ 4196.384165] Lustre: 6922:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547781949/real 1547781949] req@ffff90ab375e0000 x1622966352751184/t0(0) o39->fir-MDT0002-lwp-OST000b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781955 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [ 4196.384169] Lustre: 6922:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [ 4196.450234] Lustre: 6925:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [ 4202.385314] Lustre: 6923:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547781955/real 1547781955] req@ffff90bbf3860000 x1622966352751280/t0(0) o39->fir-MDT0000-lwp-OST0001@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781961 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [ 4202.413522] Lustre: 6923:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [ 4203.066336] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 12 seconds [ 4203.076595] LNet: 127824:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [ 4203.085943] Lustre: fir-MDT0003-lwp-OST0001: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [ 4203.101929] Lustre: Skipped 1 previous similar message [ 4203.108915] Lustre: Failing over fir-OST0001 [ 4203.874916] Lustre: server umount fir-OST0001 complete [ 4204.066396] Lustre: 6924:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1547781962/real 1547781963] req@ffff90ab62c60000 x1622966352751616/t0(0) o39->fir-MDT0003-lwp-OST0007@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781968 ref 2 fl Rpc:eXN/0/ffffffff rc 0/-1 [ 4204.066421] Lustre: fir-MDT0003-lwp-OST0005: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [ 4204.110940] Lustre: 6924:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 8 previous similar messages [ 4204.122451] Lustre: Failing over fir-OST0007 [ 4204.124045] md1: detected capacity change from 64011422924800 to 0 [ 4204.124056] md: md1 stopped. [ 4204.899040] Lustre: server umount fir-OST0007 complete [ 4205.151326] md7: detected capacity change from 64011422924800 to 0 [ 4205.157525] md: md7 stopped. [ 4206.066457] Lustre: 6920:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1547781962/real 1547781965] req@ffff90ab714f8000 x1622966352751632/t0(0) o39->fir-MDT0003-lwp-OST0009@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781968 ref 2 fl Rpc:eXN/0/ffffffff rc 0/-1 [ 4206.066459] Lustre: 6925:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1547781962/real 1547781965] req@ffff90ab375e0300 x1622966352751600/t0(0) o39->fir-MDT0003-lwp-OST0003@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781968 ref 2 fl Rpc:eXN/0/ffffffff rc 0/-1 [ 4206.068332] Lustre: Failing over fir-OST0003 [ 4206.823196] Lustre: server umount fir-OST0003 complete [ 4207.030568] md3: detected capacity change from 64011422924800 to 0 [ 4207.036762] md: md3 stopped. [ 4207.325948] md9: detected capacity change from 64011422924800 to 0 [ 4207.332149] md: md9 stopped. [ 4208.070175] Lustre: Failing over fir-OST000b [ 4208.074477] Lustre: Skipped 2 previous similar messages [ 4208.868517] Lustre: server umount fir-OST0005 complete [ 4208.873660] Lustre: Skipped 2 previous similar messages [ 4209.107451] md11: detected capacity change from 64011422924800 to 0 [ 4209.113733] md: md11 stopped. [ 4209.116915] md5: detected capacity change from 64011422924800 to 0 [ 4209.123103] md: md5 stopped. [ 4213.094582] LNet: Removed LNI 10.0.10.102@o2ib7 [ 4234.136584] md: md1 stopped. [ 4234.168589] md/raid:md1: device dm-26 operational as raid disk 0 [ 4234.174604] md/raid:md1: device dm-9 operational as raid disk 9 [ 4234.180532] md/raid:md1: device dm-16 operational as raid disk 8 [ 4234.186547] md/raid:md1: device dm-59 operational as raid disk 7 [ 4234.192560] md/raid:md1: device dm-23 operational as raid disk 6 [ 4234.198569] md/raid:md1: device dm-60 operational as raid disk 5 [ 4234.204580] md/raid:md1: device dm-34 operational as raid disk 4 [ 4234.210588] md/raid:md1: device dm-68 operational as raid disk 3 [ 4234.216594] md/raid:md1: device dm-43 operational as raid disk 2 [ 4234.222600] md/raid:md1: device dm-57 operational as raid disk 1 [ 4234.229447] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4234.247840] md1: detected capacity change from 0 to 64011422924800 [ 4234.256371] md: md5 stopped. [ 4234.297158] md/raid:md5: device dm-84 operational as raid disk 0 [ 4234.303178] md/raid:md5: device dm-8 operational as raid disk 9 [ 4234.309109] md/raid:md5: device dm-3 operational as raid disk 8 [ 4234.315040] md/raid:md5: device dm-22 operational as raid disk 7 [ 4234.321049] md/raid:md5: device dm-21 operational as raid disk 6 [ 4234.327063] md/raid:md5: device dm-67 operational as raid disk 5 [ 4234.333078] md/raid:md5: device dm-36 operational as raid disk 4 [ 4234.339094] md/raid:md5: device dm-10 operational as raid disk 3 [ 4234.345109] md/raid:md5: device dm-76 operational as raid disk 2 [ 4234.351121] md/raid:md5: device dm-56 operational as raid disk 1 [ 4234.357868] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4234.377645] md5: detected capacity change from 0 to 64011422924800 [ 4234.389218] md: md9 stopped. [ 4234.433237] md/raid:md9: device dm-47 operational as raid disk 0 [ 4234.439270] md/raid:md9: device dm-42 operational as raid disk 9 [ 4234.445308] md/raid:md9: device dm-39 operational as raid disk 8 [ 4234.451385] md/raid:md9: device dm-28 operational as raid disk 7 [ 4234.457406] md/raid:md9: device dm-116 operational as raid disk 6 [ 4234.463513] md/raid:md9: device dm-71 operational as raid disk 5 [ 4234.469539] md/raid:md9: device dm-110 operational as raid disk 4 [ 4234.475648] md/raid:md9: device dm-94 operational as raid disk 3 [ 4234.481674] md/raid:md9: device dm-33 operational as raid disk 2 [ 4234.487694] md/raid:md9: device dm-53 operational as raid disk 1 [ 4234.494684] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4234.514159] md9: detected capacity change from 0 to 64011422924800 [ 4234.572986] md: md11 stopped. [ 4234.634950] md/raid:md11: device dm-50 operational as raid disk 0 [ 4234.641060] md/raid:md11: device dm-46 operational as raid disk 9 [ 4234.647164] md/raid:md11: device dm-30 operational as raid disk 8 [ 4234.653266] md/raid:md11: device dm-112 operational as raid disk 7 [ 4234.659464] md/raid:md11: device dm-114 operational as raid disk 6 [ 4234.665664] md/raid:md11: device dm-80 operational as raid disk 5 [ 4234.671772] md/raid:md11: device dm-48 operational as raid disk 4 [ 4234.677882] md/raid:md11: device dm-40 operational as raid disk 3 [ 4234.683991] md/raid:md11: device dm-104 operational as raid disk 2 [ 4234.690188] md/raid:md11: device dm-95 operational as raid disk 1 [ 4234.697468] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4234.718053] md11: detected capacity change from 0 to 64011422924800 [ 4234.743477] md: md7 stopped. [ 4234.798982] md/raid:md7: device dm-92 operational as raid disk 0 [ 4234.805005] md/raid:md7: device dm-38 operational as raid disk 9 [ 4234.811019] md/raid:md7: device dm-49 operational as raid disk 8 [ 4234.817030] md/raid:md7: device dm-97 operational as raid disk 7 [ 4234.823046] md/raid:md7: device dm-111 operational as raid disk 6 [ 4234.829145] md/raid:md7: device dm-96 operational as raid disk 5 [ 4234.835164] md/raid:md7: device dm-101 operational as raid disk 4 [ 4234.841261] md/raid:md7: device dm-91 operational as raid disk 3 [ 4234.847280] md/raid:md7: device dm-86 operational as raid disk 2 [ 4234.853293] md/raid:md7: device dm-109 operational as raid disk 1 [ 4234.860079] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4234.905585] md7: detected capacity change from 0 to 64011422924800 [ 4234.916733] md: md3 stopped. [ 4234.975898] md/raid:md3: device dm-2 operational as raid disk 0 [ 4234.981849] md/raid:md3: device dm-0 operational as raid disk 9 [ 4234.987781] md/raid:md3: device dm-65 operational as raid disk 8 [ 4234.993798] md/raid:md3: device dm-24 operational as raid disk 7 [ 4234.999821] md/raid:md3: device dm-14 operational as raid disk 6 [ 4235.005845] md/raid:md3: device dm-118 operational as raid disk 5 [ 4235.011961] md/raid:md3: device dm-64 operational as raid disk 4 [ 4235.017989] md/raid:md3: device dm-75 operational as raid disk 3 [ 4235.024006] md/raid:md3: device dm-31 operational as raid disk 2 [ 4235.030022] md/raid:md3: device dm-15 operational as raid disk 1 [ 4235.036735] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4235.055371] md3: detected capacity change from 0 to 64011422924800 [ 4255.463825] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 4255.781716] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4256.789862] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 4257.104664] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4258.310907] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 4258.626135] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4258.856907] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 4259.167237] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4260.251949] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 4260.573059] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4266.133096] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 4266.472467] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4287.702584] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [ 4287.710213] alg: No test for adler32 (adler32-zlib) [ 4288.513109] Lustre: Lustre: Build Version: 2.12.0 [ 4288.633123] LNet: Using FastReg for registration [ 4288.648426] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [ 4292.330783] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 4292.669274] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4292.735829] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 4293.045842] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 4293.072310] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4293.237795] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 4293.384681] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4293.508719] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 4293.525792] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 4293.538850] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4293.803807] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 4293.849173] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4293.876393] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4293.887810] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 4293.976867] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 4294.101057] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4294.187890] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4294.308057] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4294.399813] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 4294.416817] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 4294.705444] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4294.718379] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4294.961833] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 4295.032503] Lustre: fir-OST0001: new disk, initializing [ 4295.257349] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4295.871384] Lustre: fir-OST0001: Not available for connect from 10.9.101.59@o2ib4 (not set up) [ 4295.871456] LustreError: 137-5: fir-OST0003_UUID: not available for connect from 10.9.101.59@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4296.037800] Lustre: srv-fir-OST0001: No data found on store. Initialize space [ 4296.151093] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [ 4297.239618] Lustre: fir-OST0005: new disk, initializing [ 4297.244902] Lustre: srv-fir-OST0005: No data found on store. Initialize space [ 4297.260352] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [ 4298.040624] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4298.040805] Lustre: fir-OST0005: Connection restored to 4cdc8cd0-fe10-8c01-71cf-daa87b9e9494 (at 10.9.101.60@o2ib4) [ 4298.040807] Lustre: Skipped 2 previous similar messages [ 4298.073678] LustreError: Skipped 4 previous similar messages [ 4298.351934] Lustre: fir-OST0007: new disk, initializing [ 4298.357163] Lustre: Skipped 3 previous similar messages [ 4298.362445] Lustre: srv-fir-OST0007: No data found on store. Initialize space [ 4298.369609] Lustre: Skipped 3 previous similar messages [ 4298.385186] Lustre: fir-OST0007: Imperative Recovery not enabled, recovery window 300-900 [ 4298.393367] Lustre: Skipped 3 previous similar messages [ 4299.444177] Lustre: fir-OST0001: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [ 4299.453398] Lustre: Skipped 2 previous similar messages [ 4299.514298] Lustre: cli-fir-OST0005-super: Allocated super-sequence [0x0000000440000400-0x0000000480000400]:5:ost] [ 4299.524668] Lustre: Skipped 1 previous similar message [ 4304.511268] Lustre: fir-OST0001: Connection restored to (at 10.0.10.52@o2ib7) [ 4304.518650] Lustre: Skipped 10 previous similar messages [ 4305.861764] LustreError: 137-5: fir-OST0006_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4305.878876] LustreError: Skipped 5 previous similar messages [ 4313.885130] Lustre: fir-OST0009: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [ 4313.895310] Lustre: Skipped 13 previous similar messages [ 4318.384864] Lustre: fir-OST0009: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [ 4318.395045] Lustre: Skipped 4 previous similar messages [ 4320.872093] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.101.59@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4320.889500] LustreError: Skipped 4 previous similar messages [ 4330.862448] Lustre: fir-OST0003: Connection restored to (at 10.9.0.2@o2ib4) [ 4330.862449] Lustre: fir-OST0005: Connection restored to (at 10.9.0.2@o2ib4) [ 4330.862452] Lustre: Skipped 1 previous similar message [ 4330.881693] Lustre: Skipped 4 previous similar messages [ 4479.769257] Lustre: fir-OST0001: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [ 4479.779521] Lustre: Skipped 12 previous similar messages [ 4951.123218] Lustre: fir-OST0005: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [ 4951.133482] Lustre: Skipped 5 previous similar messages [ 5631.455111] Lustre: fir-OST0007: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [ 6437.018609] Lustre: fir-OST0001: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [ 6437.028872] Lustre: Skipped 10 previous similar messages [54968.461575] Lustre: fir-OST0007: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [55992.483030] Lustre: fir-OST0001: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [57280.902267] Lustre: fir-OST0001: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [57280.912531] Lustre: Skipped 10 previous similar messages [57688.158300] Lustre: 9211:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547835440/real 1547835440] req@ffff90dae0740000 x1622967109624208/t0(0) o39->fir-MDT0002-lwp-OST0009@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835446 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [57688.186524] Lustre: 9211:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [57688.999323] LNet: 10636:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [57689.009473] Lustre: 10695:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1547835445/real 1547835447] req@ffff90bb90efd400 x1622967109624480/t0(0) o400->fir-MDT0001-lwp-OST000b@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835452 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [57689.009490] Lustre: fir-MDT0003-lwp-OST000b: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [57689.054229] Lustre: 10695:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [57689.999344] LNet: 10636:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 549 seconds [57690.009596] LNet: 10636:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [57690.018965] Lustre: 10688:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1547835445/real 1547835448] req@ffff90bb90ef9200 x1622967109624560/t0(0) o400->fir-MDT0003-lwp-OST0007@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835452 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [57690.018968] Lustre: fir-MDT0001-lwp-OST0007: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [57690.018969] Lustre: Skipped 7 previous similar messages [57690.068906] Lustre: 10688:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [57690.999375] LNet: 10636:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 24 seconds [57691.009541] LNet: 10636:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [57694.161443] Lustre: 9216:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547835446/real 1547835446] req@ffff90bb90efbc00 x1622967109624608/t0(0) o39->fir-MDT0000-lwp-OST0003@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835452 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [57694.166889] Lustre: Failing over fir-OST0001 [57694.193941] Lustre: 9216:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [57694.752860] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [57694.769975] LustreError: Skipped 2 previous similar messages [57694.951715] Lustre: server umount fir-OST000b complete [57694.956863] Lustre: Skipped 1 previous similar message [57695.513697] Lustre: server umount fir-OST0009 complete [57695.518842] Lustre: Skipped 1 previous similar message [57695.975281] md11: detected capacity change from 64011422924800 to 0 [57695.977724] md1: detected capacity change from 64011422924800 to 0 [57695.977733] md: md1 stopped. [57695.979891] md3: detected capacity change from 64011422924800 to 0 [57695.979894] md: md3 stopped. [57695.999709] md: md11 stopped. [57696.005889] md9: detected capacity change from 64011422924800 to 0 [57696.012088] md: md9 stopped. [57696.018538] md7: detected capacity change from 64011422924800 to 0 [57696.022955] md5: detected capacity change from 64011422924800 to 0 [57696.022964] md: md5 stopped. [57696.033785] md: md7 stopped. [57700.042599] LNet: Removed LNI 10.0.10.102@o2ib7 [57726.357117] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [57726.364797] alg: No test for adler32 (adler32-zlib) [57727.166348] Lustre: Lustre: Build Version: 2.12.0 [57727.287769] LNet: Using FastReg for registration [57727.304046] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [57732.108281] md: md11 stopped. [57732.125965] md/raid:md11: device dm-50 operational as raid disk 0 [57732.132082] md/raid:md11: device dm-46 operational as raid disk 9 [57732.138185] md/raid:md11: device dm-30 operational as raid disk 8 [57732.144289] md/raid:md11: device dm-112 operational as raid disk 7 [57732.150477] md/raid:md11: device dm-114 operational as raid disk 6 [57732.156663] md/raid:md11: device dm-80 operational as raid disk 5 [57732.162798] md/raid:md11: device dm-48 operational as raid disk 4 [57732.168891] md/raid:md11: device dm-40 operational as raid disk 3 [57732.174984] md/raid:md11: device dm-104 operational as raid disk 2 [57732.181162] md/raid:md11: device dm-95 operational as raid disk 1 [57732.188046] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [57732.206076] md11: detected capacity change from 0 to 64011422924800 [57732.220288] md: md3 stopped. [57732.240264] md/raid:md3: device dm-2 operational as raid disk 0 [57732.246193] md/raid:md3: device dm-0 operational as raid disk 9 [57732.252122] md/raid:md3: device dm-65 operational as raid disk 8 [57732.258137] md/raid:md3: device dm-24 operational as raid disk 7 [57732.264151] md/raid:md3: device dm-14 operational as raid disk 6 [57732.270171] md/raid:md3: device dm-118 operational as raid disk 5 [57732.276270] md/raid:md3: device dm-64 operational as raid disk 4 [57732.282286] md/raid:md3: device dm-75 operational as raid disk 3 [57732.288295] md/raid:md3: device dm-31 operational as raid disk 2 [57732.294305] md/raid:md3: device dm-15 operational as raid disk 1 [57732.301047] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [57732.318889] md3: detected capacity change from 0 to 64011422924800 [57732.332939] md: md5 stopped. [57732.354430] md/raid:md5: device dm-84 operational as raid disk 0 [57732.360455] md/raid:md5: device dm-8 operational as raid disk 9 [57732.366379] md/raid:md5: device dm-3 operational as raid disk 8 [57732.372311] md/raid:md5: device dm-22 operational as raid disk 7 [57732.378320] md/raid:md5: device dm-21 operational as raid disk 6 [57732.384339] md/raid:md5: device dm-67 operational as raid disk 5 [57732.390349] md/raid:md5: device dm-36 operational as raid disk 4 [57732.396367] md/raid:md5: device dm-10 operational as raid disk 3 [57732.402379] md/raid:md5: device dm-76 operational as raid disk 2 [57732.408394] md/raid:md5: device dm-56 operational as raid disk 1 [57732.416425] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [57732.451307] md5: detected capacity change from 0 to 64011422924800 [57732.463814] md: md1 stopped. [57732.485629] md/raid:md1: device dm-26 operational as raid disk 0 [57732.491657] md/raid:md1: device dm-9 operational as raid disk 9 [57732.497591] md/raid:md1: device dm-16 operational as raid disk 8 [57732.503613] md/raid:md1: device dm-59 operational as raid disk 7 [57732.509625] md/raid:md1: device dm-23 operational as raid disk 6 [57732.515641] md/raid:md1: device dm-60 operational as raid disk 5 [57732.521656] md/raid:md1: device dm-34 operational as raid disk 4 [57732.527671] md/raid:md1: device dm-68 operational as raid disk 3 [57732.533681] md/raid:md1: device dm-43 operational as raid disk 2 [57732.539688] md/raid:md1: device dm-57 operational as raid disk 1 [57732.546430] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [57732.565463] md1: detected capacity change from 0 to 64011422924800 [57732.581018] md: md9 stopped. [57732.626999] md/raid:md9: device dm-47 operational as raid disk 0 [57732.633019] md/raid:md9: device dm-42 operational as raid disk 9 [57732.639031] md/raid:md9: device dm-39 operational as raid disk 8 [57732.645047] md/raid:md9: device dm-28 operational as raid disk 7 [57732.651059] md/raid:md9: device dm-116 operational as raid disk 6 [57732.657161] md/raid:md9: device dm-71 operational as raid disk 5 [57732.663178] md/raid:md9: device dm-110 operational as raid disk 4 [57732.669276] md/raid:md9: device dm-94 operational as raid disk 3 [57732.675311] md/raid:md9: device dm-33 operational as raid disk 2 [57732.681326] md/raid:md9: device dm-53 operational as raid disk 1 [57732.688089] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [57732.706049] md9: detected capacity change from 0 to 64011422924800 [57732.720110] md: md7 stopped. [57732.761722] md/raid:md7: device dm-92 operational as raid disk 0 [57732.767735] md/raid:md7: device dm-38 operational as raid disk 9 [57732.773778] md/raid:md7: device dm-49 operational as raid disk 8 [57732.779810] md/raid:md7: device dm-97 operational as raid disk 7 [57732.785833] md/raid:md7: device dm-111 operational as raid disk 6 [57732.791942] md/raid:md7: device dm-96 operational as raid disk 5 [57732.797964] md/raid:md7: device dm-101 operational as raid disk 4 [57732.804080] md/raid:md7: device dm-91 operational as raid disk 3 [57732.810115] md/raid:md7: device dm-86 operational as raid disk 2 [57732.816143] md/raid:md7: device dm-109 operational as raid disk 1 [57732.824522] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [57732.843676] md7: detected capacity change from 0 to 64011422924800 [57733.079639] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [57733.159638] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [57733.394834] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57733.412637] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [57733.484620] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57733.756020] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57733.845647] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [57733.853651] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [57734.009649] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [57734.190827] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57734.230209] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57734.318745] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57737.287552] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [57740.287637] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [57742.287685] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 5 seconds [57742.297767] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [57743.290040] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [57743.307106] Lustre: fir-OST000b: in recovery but waiting for the first client to connect [57745.287764] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 8 seconds [57745.297844] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [57746.344402] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [57746.361522] LustreError: Skipped 5 previous similar messages [57747.625775] Lustre: fir-OST000b: Will be in recovery for at least 5:00, or until 5 clients reconnect [57747.635145] Lustre: fir-OST000b: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [57748.166074] Lustre: fir-OST000b: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [57748.176520] Lustre: Skipped 1 previous similar message [57748.290097] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [57748.298898] Lustre: fir-OST0003: in recovery but waiting for the first client to connect [57750.287886] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 13 seconds [57750.298054] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [57751.822840] LustreError: 137-5: fir-OST0001_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [57751.839984] Lustre: fir-OST0003: Will be in recovery for at least 5:00, or until 5 clients reconnect [57751.849172] Lustre: fir-OST0003: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [57753.290215] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [57753.298995] Lustre: fir-OST0005: in recovery but waiting for the first client to connect [57754.010996] Lustre: fir-OST0003: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [57754.011095] Lustre: fir-OST0005: Will be in recovery for at least 5:00, or until 5 clients reconnect [57754.030559] Lustre: Skipped 1 previous similar message [57754.487311] LustreError: 137-5: fir-OST0001_UUID: not available for connect from 10.9.101.59@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [57754.504726] LustreError: Skipped 3 previous similar messages [57756.593854] LustreError: 137-5: fir-OST0001_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [57756.611221] LustreError: Skipped 3 previous similar messages [57759.288117] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 22 seconds [57759.298286] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [57759.309970] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [57759.318810] Lustre: fir-OST0009: in recovery but waiting for the first client to connect [57761.445287] Lustre: fir-OST0009: Will be in recovery for at least 5:00, or until 5 clients reconnect [57761.454450] Lustre: fir-OST0009: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [57761.464625] Lustre: Skipped 6 previous similar messages [57764.290626] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [57764.299439] Lustre: fir-OST0001: in recovery but waiting for the first client to connect [57769.351628] Lustre: fir-OST0007: Will be in recovery for at least 5:00, or until 5 clients reconnect [57769.360761] Lustre: Skipped 1 previous similar message [57771.345085] Lustre: fir-OST0005: Connection restored to (at 10.9.0.2@o2ib4) [57771.352157] Lustre: Skipped 13 previous similar messages [57771.394790] Lustre: fir-OST0005: Recovery over after 0:17, of 5 clients 5 recovered and 0 were evicted. [57771.899082] Lustre: fir-OST0007: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 5 known clients (2 recovered, 0 in progress, and 0 evicted) already passed deadline 0:02 [57772.851558] Lustre: fir-OST0007: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 5 known clients (3 recovered, 0 in progress, and 0 evicted) already passed deadline 0:03 [57772.870515] Lustre: Skipped 3 previous similar messages [57773.606049] Lustre: fir-OST0007: Recovery over after 0:04, of 5 clients 5 recovered and 0 were evicted. [57773.615446] Lustre: Skipped 4 previous similar messages [57797.077615] Lustre: fir-OST0007: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [57797.086863] Lustre: Skipped 24 previous similar messages [57823.398641] Lustre: fir-OST0001: deleting orphan objects from 0x740000400:3 to 0x740000400:33 [57824.050626] Lustre: fir-OST0003: deleting orphan objects from 0xcc0000402:23 to 0xcc0000402:65 [57824.079282] Lustre: fir-OST0005: deleting orphan objects from 0x440000402:23 to 0x440000402:65 [57824.081890] Lustre: fir-OST0009: deleting orphan objects from 0x640000402:23 to 0x640000402:65 [57824.083876] Lustre: fir-OST0001: deleting orphan objects from 0x740000402:23 to 0x740000402:65 [57824.117607] Lustre: fir-OST0007: deleting orphan objects from 0xe80000402:24 to 0xe80000402:65 [57824.178042] Lustre: fir-OST000b: deleting orphan objects from 0x8c0000402:23 to 0x8c0000402:65 [58646.418425] Lustre: fir-OST0001: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [58646.428703] Lustre: Skipped 7 previous similar messages [58756.938394] Lustre: fir-OST0007: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [58756.948661] Lustre: Skipped 6 previous similar messages [58757.137881] Lustre: 12440:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0007: credits 8227 > trans_max 8192 [58757.147799] Lustre: 12440:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [58757.157195] Lustre: 12440:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [58757.167042] Lustre: 12440:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [58757.177559] Lustre: 12440:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [58757.186871] Lustre: 12440:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [58757.196361] Pid: 12440, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [58757.206638] Call Trace: [58757.209109] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [58757.215681] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [58757.222706] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [58757.229705] [] ofd_trans_start+0x75/0xf0 [ofd] [58757.235856] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [58757.242506] [] ofd_commitrw+0x4b2/0xa10 [ofd] [58757.248560] [] obd_commitrw+0x9c/0x370 [ptlrpc] [58757.254821] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [58757.261431] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [58757.268378] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [58757.276105] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [58757.282433] [] kthread+0xd1/0xe0 [58757.287348] [] ret_from_fork_nospec_begin+0xe/0x21 [58757.293848] [] 0xffffffffffffffff [58757.298945] Lustre: 12440:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0007: opcode 7: before 8192 < left 8209, rollback = 7 [58757.705109] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [58757.714501] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 10 previous similar messages [58757.724358] Lustre: 12443:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [58757.734186] Lustre: 12443:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 10 previous similar messages [58757.744046] Lustre: 12443:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [58757.754575] Lustre: 12443:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 10 previous similar messages [58757.764411] Lustre: 12443:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [58757.773740] Lustre: 12443:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 10 previous similar messages [58757.783574] Lustre: 12443:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [58757.793057] Lustre: 12443:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 10 previous similar messages [58757.805371] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0007: opcode 7: before 8192 < left 8209, rollback = 7 [58757.817108] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 10 previous similar messages [58758.714825] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [58758.724219] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 29 previous similar messages [58758.734074] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [58758.743925] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 29 previous similar messages [58758.753767] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [58758.764319] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 29 previous similar messages [58758.774144] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [58758.783451] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 29 previous similar messages [58758.793301] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [58758.802797] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 29 previous similar messages [58758.815544] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0007: opcode 7: before 8192 < left 8209, rollback = 7 [58758.827281] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 29 previous similar messages [58931.899516] Lustre: fir-OST0001: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [58971.399382] Lustre: fir-OST0005: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [58971.539486] Lustre: 12437:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0005: opcode 7: before 8192 < left 8209, rollback = 7 [58971.551243] Lustre: 12437:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 24 previous similar messages [58971.560901] Lustre: 12437:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [58971.570344] Lustre: 12437:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 25 previous similar messages [58971.580210] Lustre: 12437:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [58971.590040] Lustre: 12437:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 25 previous similar messages [58971.599895] Lustre: 12437:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [58971.610422] Lustre: 12437:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 25 previous similar messages [58971.620302] Lustre: 12437:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [58971.629608] Lustre: 12437:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 25 previous similar messages [58971.639457] Lustre: 12437:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [58971.648936] Lustre: 12437:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 25 previous similar messages [59004.961207] Lustre: fir-OST0009: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [59004.971500] Lustre: Skipped 2 previous similar messages [59005.169534] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [59005.181266] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 59 previous similar messages [59005.190921] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59005.200318] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 59 previous similar messages [59005.210161] Lustre: 12443:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59005.219991] Lustre: 12443:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 59 previous similar messages [59005.229823] Lustre: 12443:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59005.240371] Lustre: 12443:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 59 previous similar messages [59005.250211] Lustre: 12443:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59005.259522] Lustre: 12443:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 59 previous similar messages [59005.269367] Lustre: 12443:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59005.278844] Lustre: 12443:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 59 previous similar messages [59013.245145] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [59013.256886] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 151 previous similar messages [59013.266651] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59013.276051] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 151 previous similar messages [59013.286002] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59013.295835] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 151 previous similar messages [59013.305782] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59013.316302] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 151 previous similar messages [59013.326228] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59013.335537] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 151 previous similar messages [59013.345469] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59013.354955] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 151 previous similar messages [59029.292666] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [59029.304404] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 291 previous similar messages [59029.314150] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59029.323544] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 291 previous similar messages [59029.333480] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59029.343304] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 291 previous similar messages [59029.353222] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59029.363754] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 291 previous similar messages [59029.373678] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59029.382978] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 291 previous similar messages [59029.392911] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59029.402394] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 291 previous similar messages [59061.322559] Lustre: 12437:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [59061.334290] Lustre: 12437:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 541 previous similar messages [59061.344032] Lustre: 12437:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59061.353430] Lustre: 12437:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 541 previous similar messages [59061.363361] Lustre: 12437:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59061.373188] Lustre: 12437:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 541 previous similar messages [59061.383121] Lustre: 12437:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59061.393639] Lustre: 12437:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 541 previous similar messages [59061.403574] Lustre: 12437:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59061.412880] Lustre: 12437:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 541 previous similar messages [59061.422803] Lustre: 12437:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59061.432288] Lustre: 12437:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 541 previous similar messages [59105.322022] Lustre: 24842:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0009: credits 8231 > trans_max 8192 [59105.331941] Pid: 24842, comm: ll_ost_io01_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59105.342200] Call Trace: [59105.344698] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59105.351271] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59105.358299] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59105.365303] [] ofd_trans_start+0x75/0xf0 [ofd] [59105.371445] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59105.378096] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59105.384152] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59105.390412] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59105.397020] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59105.403968] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59105.411717] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59105.418050] [] kthread+0xd1/0xe0 [59105.422979] [] ret_from_fork_nospec_begin+0xe/0x21 [59105.429457] [] 0xffffffffffffffff [59125.411401] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [59125.423139] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1081 previous similar messages [59125.432987] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59125.442378] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1082 previous similar messages [59125.452413] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59125.462243] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1082 previous similar messages [59125.472259] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59125.482773] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1082 previous similar messages [59125.492795] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59125.502100] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1082 previous similar messages [59125.512120] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59125.521606] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1082 previous similar messages [59154.333318] Lustre: fir-OST0009: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [59154.343580] Lustre: Skipped 1 previous similar message [59165.487735] Lustre: 27528:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0009: credits 8227 > trans_max 8192 [59165.497655] Pid: 27528, comm: ll_ost_io01_005 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59165.507918] Call Trace: [59165.510392] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59165.516965] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59165.524001] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59165.531005] [] ofd_trans_start+0x75/0xf0 [ofd] [59165.537170] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59165.543840] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59165.549909] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59165.556171] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59165.562782] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59165.569727] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59165.577456] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59165.583786] [] kthread+0xd1/0xe0 [59165.588714] [] ret_from_fork_nospec_begin+0xe/0x21 [59165.595191] [] 0xffffffffffffffff [59362.903225] Lustre: fir-OST0009: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [59362.913500] Lustre: Skipped 7 previous similar messages [59374.742731] Lustre: 28622:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0007: opcode 7: before 8192 < left 8209, rollback = 7 [59374.754467] Lustre: 28622:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1941 previous similar messages [59374.764296] Lustre: 28622:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59374.773687] Lustre: 28622:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1942 previous similar messages [59374.783721] Lustre: 28622:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59374.793559] Lustre: 28622:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1942 previous similar messages [59374.803555] Lustre: 28622:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59374.814093] Lustre: 28622:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1942 previous similar messages [59374.824095] Lustre: 28622:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59374.824306] Lustre: 24860:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0007: credits 8207 > trans_max 8192 [59374.824314] Lustre: 24860:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59374.824317] Lustre: 24860:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1942 previous similar messages [59374.824319] Pid: 24860, comm: ll_ost_io02_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59374.824319] Call Trace: [59374.824342] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59374.824348] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59374.824358] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59374.824366] [] ofd_trans_start+0x75/0xf0 [ofd] [59374.824372] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59374.824378] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59374.824435] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59374.824470] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59374.824508] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59374.824540] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59374.824571] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59374.824577] [] kthread+0xd1/0xe0 [59374.824582] [] ret_from_fork_nospec_begin+0xe/0x21 [59374.824610] [] 0xffffffffffffffff [59374.964472] Lustre: 28622:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1943 previous similar messages [59434.881775] Lustre: 12437:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0001: credits 8227 > trans_max 8192 [59434.891690] Pid: 12437, comm: ll_ost_io00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59434.901946] Call Trace: [59434.904412] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59434.910984] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59434.918010] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59434.925017] [] ofd_trans_start+0x75/0xf0 [ofd] [59434.931158] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59434.937809] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59434.943863] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59434.950132] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59434.956734] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59434.963673] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59434.971400] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59434.977742] [] kthread+0xd1/0xe0 [59434.982658] [] ret_from_fork_nospec_begin+0xe/0x21 [59434.989142] [] 0xffffffffffffffff [59434.994184] Pid: 27528, comm: ll_ost_io01_005 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59435.004444] Call Trace: [59435.006921] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59435.013499] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59435.020511] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59435.027551] [] ofd_trans_start+0x75/0xf0 [ofd] [59435.033726] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59435.040410] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59435.046512] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59435.052787] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59435.059428] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59435.066404] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59435.074162] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59435.080493] [] kthread+0xd1/0xe0 [59435.085434] [] ret_from_fork_nospec_begin+0xe/0x21 [59435.091909] [] 0xffffffffffffffff [59538.165767] Lustre: 12440:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0001: credits 8231 > trans_max 8192 [59538.175685] Lustre: 12440:0:(osd_handler.c:1870:osd_trans_start()) Skipped 1 previous similar message [59538.184933] Pid: 12440, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59538.195223] Call Trace: [59538.197701] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59538.204263] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59538.211272] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59538.218286] [] ofd_trans_start+0x75/0xf0 [ofd] [59538.224415] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59538.231090] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59538.237136] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59538.243399] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59538.249990] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59538.256951] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59538.264662] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59538.271002] [] kthread+0xd1/0xe0 [59538.275920] [] ret_from_fork_nospec_begin+0xe/0x21 [59538.282411] [] 0xffffffffffffffff [59730.732331] Lustre: fir-OST0005: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [59730.742598] Lustre: Skipped 5 previous similar messages [61282.378607] Lustre: fir-OST000b: haven't heard from client e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) in 228 seconds. I think it's dead, and I am evicting it. exp ffff90bbfaf36000, cur 1547839040 expire 1547838890 last 1547838812 [64029.166012] Lustre: fir-OST0001: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [64029.176459] Lustre: Skipped 8 previous similar messages [64099.970141] Lustre: fir-OST0001: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [64099.980576] Lustre: Skipped 5 previous similar messages [64110.384762] Lustre: 12437:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0005: credits 8227 > trans_max 8192 [64110.394680] Lustre: 12437:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64110.404110] Lustre: 12437:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 2007 previous similar messages [64110.414167] Lustre: 12437:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64110.423997] Lustre: 12437:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 2007 previous similar messages [64110.434034] Lustre: 12437:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64110.444577] Lustre: 12437:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 2007 previous similar messages [64110.454579] Lustre: 12437:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64110.463910] Lustre: 12437:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 2006 previous similar messages [64110.473923] Lustre: 12437:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64110.483405] Lustre: 12437:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 2007 previous similar messages [64110.493435] Pid: 12437, comm: ll_ost_io00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64110.503690] Call Trace: [64110.506157] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64110.512729] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64110.519740] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64110.526742] [] ofd_trans_start+0x75/0xf0 [ofd] [64110.532885] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64110.539534] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64110.545590] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64110.551867] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64110.558488] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64110.565447] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64110.573185] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64110.579523] [] kthread+0xd1/0xe0 [64110.584437] [] ret_from_fork_nospec_begin+0xe/0x21 [64110.590929] [] 0xffffffffffffffff [64110.596004] Lustre: 12437:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0005: opcode 7: before 8192 < left 8209, rollback = 7 [64110.607762] Lustre: 12437:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 2003 previous similar messages [64146.847095] Lustre: 12446:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [64146.858852] Lustre: 12446:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 59 previous similar messages [64146.868508] Lustre: 12446:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64146.877923] Lustre: 12446:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 60 previous similar messages [64146.887788] Lustre: 12446:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64146.897633] Lustre: 12446:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 60 previous similar messages [64146.907462] Lustre: 12446:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64146.917995] Lustre: 12446:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 60 previous similar messages [64146.927818] Lustre: 12446:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64146.937126] Lustre: 12446:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 60 previous similar messages [64146.946954] Lustre: 12446:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64146.956434] Lustre: 12446:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 60 previous similar messages [64211.087668] Lustre: 12446:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [64211.099408] Lustre: 12446:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1011 previous similar messages [64211.109242] Lustre: 12446:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64211.118633] Lustre: 12446:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1011 previous similar messages [64211.128635] Lustre: 12446:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64211.138482] Lustre: 12446:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1011 previous similar messages [64211.148481] Lustre: 12446:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64211.159020] Lustre: 12446:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1011 previous similar messages [64211.169021] Lustre: 12446:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64211.178344] Lustre: 12446:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1011 previous similar messages [64211.188340] Lustre: 12446:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64211.197822] Lustre: 12446:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1011 previous similar messages [64241.831622] Lustre: 12446:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0009: credits 8231 > trans_max 8192 [64241.841539] Pid: 12446, comm: ll_ost_io03_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64241.851833] Call Trace: [64241.854305] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64241.860875] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64241.867886] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64241.874889] [] ofd_trans_start+0x75/0xf0 [ofd] [64241.881040] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64241.887692] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64241.893754] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64241.900044] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64241.906660] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64241.913604] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64241.921333] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64241.927662] [] kthread+0xd1/0xe0 [64241.932593] [] ret_from_fork_nospec_begin+0xe/0x21 [64241.939068] [] 0xffffffffffffffff [64301.996032] Lustre: 24842:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0009: credits 8227 > trans_max 8192 [64302.005950] Pid: 24842, comm: ll_ost_io01_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64302.016206] Call Trace: [64302.018678] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64302.025250] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64302.032260] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64302.039256] [] ofd_trans_start+0x75/0xf0 [ofd] [64302.045398] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64302.052048] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64302.058102] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64302.064378] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64302.070983] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64302.077920] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64302.085648] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64302.091976] [] kthread+0xd1/0xe0 [64302.096906] [] ret_from_fork_nospec_begin+0xe/0x21 [64302.103383] [] 0xffffffffffffffff [64339.108497] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [64339.120236] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1919 previous similar messages [64339.130069] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64339.139460] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1921 previous similar messages [64339.149479] Lustre: 12443:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64339.159311] Lustre: 12443:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1921 previous similar messages [64339.169325] Lustre: 12443:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64339.179847] Lustre: 12443:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1921 previous similar messages [64339.189882] Lustre: 12443:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64339.199194] Lustre: 12443:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1921 previous similar messages [64339.209227] Lustre: 12443:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64339.218725] Lustre: 12443:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1921 previous similar messages [64442.971309] Lustre: 12446:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0003: credits 8231 > trans_max 8192 [64442.981229] Pid: 12446, comm: ll_ost_io03_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64442.991483] Call Trace: [64442.993960] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64443.000529] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64443.007560] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64443.014593] [] ofd_trans_start+0x75/0xf0 [ofd] [64443.020749] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64443.027405] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64443.033483] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64443.039757] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64443.046365] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64443.053319] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64443.061036] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64443.067385] [] kthread+0xd1/0xe0 [64443.072301] [] ret_from_fork_nospec_begin+0xe/0x21 [64443.078788] [] 0xffffffffffffffff [64503.253878] Lustre: 12437:0:(osd_handler.c:1870:osd_trans_start()) fir-OST000b: credits 8227 > trans_max 8192 [64503.263801] Pid: 12437, comm: ll_ost_io00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64503.274087] Call Trace: [64503.276560] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64503.283136] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64503.290166] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64503.297191] [] ofd_trans_start+0x75/0xf0 [ofd] [64503.303327] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64503.309975] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64503.316035] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64503.322315] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64503.328905] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64503.335846] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64503.343560] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64503.349906] [] kthread+0xd1/0xe0 [64503.354818] [] ret_from_fork_nospec_begin+0xe/0x21 [64503.361294] [] 0xffffffffffffffff [64503.366323] Pid: 24860, comm: ll_ost_io02_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64503.376600] Call Trace: [64503.379070] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64503.385639] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64503.392631] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64503.399634] [] ofd_trans_start+0x75/0xf0 [ofd] [64503.405803] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64503.412454] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64503.418524] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64503.424795] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64503.431394] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64503.438331] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64503.446062] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64503.452406] [] kthread+0xd1/0xe0 [64503.457337] [] ret_from_fork_nospec_begin+0xe/0x21 [64503.463813] [] 0xffffffffffffffff [64595.204308] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST000b: opcode 7: before 8192 < left 8209, rollback = 7 [64595.216043] Lustre: 24842:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 3633 previous similar messages [64595.225874] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64595.235265] Lustre: 24842:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 3636 previous similar messages [64595.239563] Lustre: 28622:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64595.239566] Lustre: 28622:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 3636 previous similar messages [64595.239568] Lustre: 28622:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64595.239570] Lustre: 28622:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 3636 previous similar messages [64595.239572] Lustre: 28622:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64595.239574] Lustre: 28622:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 3636 previous similar messages [64595.239576] Lustre: 28622:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64595.239577] Lustre: 28622:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 3636 previous similar messages [64690.722161] Lustre: 12437:0:(osd_handler.c:1870:osd_trans_start()) fir-OST000b: credits 8231 > trans_max 8192 [64690.732076] Lustre: 12437:0:(osd_handler.c:1870:osd_trans_start()) Skipped 1 previous similar message [64690.741304] Pid: 12437, comm: ll_ost_io00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64690.751585] Call Trace: [64690.754054] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64690.760628] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64690.767647] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64690.774772] [] ofd_trans_start+0x75/0xf0 [ofd] [64690.780908] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64690.787614] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64690.793656] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64690.799939] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64690.806537] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64690.813493] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64690.821209] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64690.827553] [] kthread+0xd1/0xe0 [64690.832469] [] ret_from_fork_nospec_begin+0xe/0x21 [64690.838955] [] 0xffffffffffffffff [64751.073416] Pid: 12440, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64751.083673] Call Trace: [64751.086138] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64751.092729] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64751.099756] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64751.106760] [] ofd_trans_start+0x75/0xf0 [ofd] [64751.112912] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64751.119591] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64751.125653] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64751.131914] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64751.138539] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64751.145486] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64751.153214] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64751.159541] [] kthread+0xd1/0xe0 [64751.164470] [] ret_from_fork_nospec_begin+0xe/0x21 [64751.170947] [] 0xffffffffffffffff [64751.175991] Pid: 24842, comm: ll_ost_io01_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64751.186250] Call Trace: [64751.188716] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64751.195277] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64751.202330] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64751.209326] [] ofd_trans_start+0x75/0xf0 [ofd] [64751.215468] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64751.222116] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64751.228172] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64751.234440] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64751.241042] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64751.248002] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64751.255733] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64751.262061] [] kthread+0xd1/0xe0 [64751.266969] [] ret_from_fork_nospec_begin+0xe/0x21 [64751.273449] [] 0xffffffffffffffff [64887.074090] Lustre: 28622:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0003: credits 8231 > trans_max 8192 [64887.084005] Lustre: 28622:0:(osd_handler.c:1870:osd_trans_start()) Skipped 2 previous similar messages [64887.093349] Pid: 28622, comm: ll_ost_io00_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64887.103658] Call Trace: [64887.106125] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64887.112698] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64887.119726] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64887.126733] [] ofd_trans_start+0x75/0xf0 [ofd] [64887.132900] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64887.139549] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64887.145603] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64887.151873] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64887.158491] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64887.165437] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64887.173165] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64887.179494] [] kthread+0xd1/0xe0 [64887.184408] [] ret_from_fork_nospec_begin+0xe/0x21 [64887.190905] [] 0xffffffffffffffff [64947.232224] Pid: 12437, comm: ll_ost_io00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64947.242509] Call Trace: [64947.244975] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64947.251548] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64947.258574] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64947.265598] [] ofd_trans_start+0x75/0xf0 [ofd] [64947.271749] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64947.278398] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64947.284439] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64947.290706] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64947.297300] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64947.304244] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64947.311957] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64947.318285] [] kthread+0xd1/0xe0 [64947.323198] [] ret_from_fork_nospec_begin+0xe/0x21 [64947.329672] [] 0xffffffffffffffff [64947.334703] Pid: 88869, comm: ll_ost_io03_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64947.344977] Call Trace: [64947.347462] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64947.354033] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64947.361043] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64947.368065] [] ofd_trans_start+0x75/0xf0 [ofd] [64947.374207] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64947.380860] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64947.386932] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64947.393201] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64947.399811] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64947.406746] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64947.414474] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64947.420803] [] kthread+0xd1/0xe0 [64947.425734] [] ret_from_fork_nospec_begin+0xe/0x21 [64947.432209] [] 0xffffffffffffffff [65007.674217] Pid: 91673, comm: ll_ost_io02_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65007.684473] Call Trace: [65007.686940] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65007.693512] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65007.700526] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65007.707532] [] ofd_trans_start+0x75/0xf0 [ofd] [65007.713684] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65007.720337] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65007.726399] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65007.732686] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65007.739313] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65007.746261] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65007.754025] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65007.760351] [] kthread+0xd1/0xe0 [65007.765265] [] ret_from_fork_nospec_begin+0xe/0x21 [65007.771742] [] 0xffffffffffffffff [65067.785641] Lustre: 12437:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0003: credits 8227 > trans_max 8192 [65067.795592] Lustre: 12437:0:(osd_handler.c:1870:osd_trans_start()) Skipped 3 previous similar messages [65067.804911] Pid: 12437, comm: ll_ost_io00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65067.815203] Call Trace: [65067.817667] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65067.824240] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65067.831252] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65067.838262] [] ofd_trans_start+0x75/0xf0 [ofd] [65067.844406] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65067.851058] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65067.857112] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65067.863381] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65067.869983] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65067.876919] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65067.884648] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65067.890976] [] kthread+0xd1/0xe0 [65067.895891] [] ret_from_fork_nospec_begin+0xe/0x21 [65067.902373] [] 0xffffffffffffffff [65107.413377] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0003: opcode 7: before 8192 < left 8209, rollback = 7 [65107.425113] Lustre: 12443:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 7367 previous similar messages [65107.434944] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [65107.439343] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [65107.439345] Lustre: 24842:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 7375 previous similar messages [65107.439348] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [65107.439350] Lustre: 24842:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 7375 previous similar messages [65107.439352] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [65107.439354] Lustre: 24842:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 7375 previous similar messages [65107.439356] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [65107.439357] Lustre: 24842:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 7375 previous similar messages [65107.523444] Lustre: 12443:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 7375 previous similar messages [65174.116143] Pid: 100073, comm: ll_ost_io02_005 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65174.126489] Call Trace: [65174.128958] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65174.135529] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65174.142541] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65174.149542] [] ofd_trans_start+0x75/0xf0 [ofd] [65174.155684] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65174.162336] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65174.168408] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65174.174702] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65174.181299] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65174.188242] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65174.195970] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65174.202328] [] kthread+0xd1/0xe0 [65174.207266] [] ret_from_fork_nospec_begin+0xe/0x21 [65174.213738] [] 0xffffffffffffffff [65234.326187] Pid: 12446, comm: ll_ost_io03_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65234.336450] Call Trace: [65234.338923] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65234.345493] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65234.352509] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65234.359537] [] ofd_trans_start+0x75/0xf0 [ofd] [65234.365695] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65234.372343] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65234.378402] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65234.384694] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65234.391306] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65234.398249] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65234.405979] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65234.412315] [] kthread+0xd1/0xe0 [65234.417228] [] ret_from_fork_nospec_begin+0xe/0x21 [65234.423721] [] 0xffffffffffffffff [65234.428752] Pid: 100073, comm: ll_ost_io02_005 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65234.439095] Call Trace: [65234.441562] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65234.448133] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65234.455130] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65234.462123] [] ofd_trans_start+0x75/0xf0 [ofd] [65234.468293] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65234.474959] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65234.481011] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65234.487290] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65234.493895] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65234.500828] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65234.508556] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65234.514901] [] kthread+0xd1/0xe0 [65234.519833] [] ret_from_fork_nospec_begin+0xe/0x21 [65234.526310] [] 0xffffffffffffffff [65707.516152] Lustre: 92904:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0009: opcode 7: before 8192 < left 8209, rollback = 7 [65707.527893] Lustre: 92904:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 8683 previous similar messages [65707.537725] Lustre: 92904:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [65707.547117] Lustre: 92904:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 8684 previous similar messages [65707.557136] Lustre: 92904:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [65707.566960] Lustre: 92904:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 8686 previous similar messages [65707.576974] Lustre: 92904:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [65707.587493] Lustre: 92904:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 8686 previous similar messages [65707.597495] Lustre: 92904:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [65707.606818] Lustre: 92904:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 8686 previous similar messages [65707.616814] Lustre: 92904:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [65707.626298] Lustre: 92904:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 8686 previous similar messages [73879.640894] Lustre: fir-OST0001: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [73879.651343] Lustre: Skipped 3 previous similar messages [81474.714173] Lustre: fir-OST0001: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [81474.724460] Lustre: Skipped 5 previous similar messages [81487.064475] Lustre: 30751:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST000b: opcode 7: before 8192 < left 8209, rollback = 7 [81487.076227] Lustre: 30751:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 69 previous similar messages [81487.085882] Lustre: 30751:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [81487.095274] Lustre: 30751:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 69 previous similar messages [81487.105130] Lustre: 30751:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [81487.114989] Lustre: 30751:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 69 previous similar messages [81487.124822] Lustre: 30751:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [81487.135359] Lustre: 30751:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 69 previous similar messages [81487.145189] Lustre: 30751:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [81487.154510] Lustre: 30751:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 69 previous similar messages [81487.164342] Lustre: 30751:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [81487.173822] Lustre: 30751:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 69 previous similar messages [81562.158766] Lustre: 100073:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0003: opcode 7: before 8192 < left 8209, rollback = 7 [81562.170584] Lustre: 100073:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1755 previous similar messages [81562.180507] Lustre: 100073:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [81562.189993] Lustre: 100073:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1755 previous similar messages [81562.200103] Lustre: 100073:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [81562.210010] Lustre: 100073:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1755 previous similar messages [81562.220109] Lustre: 100073:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [81562.230733] Lustre: 100073:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1755 previous similar messages [81562.240840] Lustre: 100073:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [81562.250234] Lustre: 100073:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1755 previous similar messages [81562.260328] Lustre: 100073:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [81562.269905] Lustre: 100073:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1755 previous similar messages [81669.328494] Lustre: 24842:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0003: credits 8231 > trans_max 8192 [81669.338407] Lustre: 24842:0:(osd_handler.c:1870:osd_trans_start()) Skipped 3 previous similar messages [81669.347720] Pid: 24842, comm: ll_ost_io01_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [81669.357975] Call Trace: [81669.360448] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [81669.367022] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [81669.374016] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [81669.381057] [] ofd_trans_start+0x75/0xf0 [ofd] [81669.387217] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [81669.393865] [] ofd_commitrw+0x4b2/0xa10 [ofd] [81669.399930] [] obd_commitrw+0x9c/0x370 [ptlrpc] [81669.406205] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [81669.412817] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [81669.419753] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [81669.427503] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [81669.433836] [] kthread+0xd1/0xe0 [81669.438749] [] ret_from_fork_nospec_begin+0xe/0x21 [81669.445240] [] 0xffffffffffffffff [81712.186446] Lustre: 12440:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0003: opcode 7: before 8192 < left 8209, rollback = 7 [81712.198181] Lustre: 12440:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 2469 previous similar messages [81712.208007] Lustre: 12440:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [81712.217399] Lustre: 12440:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 2470 previous similar messages [81712.227435] Lustre: 12440:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [81712.237261] Lustre: 12440:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 2470 previous similar messages [81712.247274] Lustre: 12440:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [81712.257794] Lustre: 12440:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 2470 previous similar messages [81712.267816] Lustre: 12440:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [81712.277139] Lustre: 12440:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 2470 previous similar messages [81712.287143] Lustre: 12440:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [81712.296650] Lustre: 12440:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 2470 previous similar messages [81729.508228] Lustre: 12440:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0003: credits 8227 > trans_max 8192 [81729.518153] Pid: 12440, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [81729.528433] Call Trace: [81729.530902] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [81729.537470] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [81729.544499] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [81729.551520] [] ofd_trans_start+0x75/0xf0 [ofd] [81729.557662] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [81729.564313] [] ofd_commitrw+0x4b2/0xa10 [ofd] [81729.570380] [] obd_commitrw+0x9c/0x370 [ptlrpc] [81729.576636] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [81729.583246] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [81729.590192] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [81729.597920] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [81729.604251] [] kthread+0xd1/0xe0 [81729.609179] [] ret_from_fork_nospec_begin+0xe/0x21 [81729.615657] [] 0xffffffffffffffff [81811.115731] Lustre: 10177:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547859561/real 1547859561] req@ffff90cb9b525a00 x1623023141907408/t0(0) o400->fir-MDT0001-lwp-OST0007@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547859568 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [81811.115738] Lustre: fir-MDT0003-lwp-OST0007: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [81811.115740] Lustre: Skipped 2 previous similar messages [81811.165306] Lustre: 10177:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 14 previous similar messages [81812.903774] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [81812.913859] LNet: 10084:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 9 previous similar messages [81813.325784] Lustre: 36794:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547859564/real 1547859564] req@ffff90a3333a8300 x1623023141907424/t0(0) o39->fir-MDT0002-lwp-OST0009@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547859570 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [81813.344131] Lustre: Failing over fir-OST0003 [81813.358386] Lustre: 36794:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [81813.905706] Lustre: Failing over fir-OST0001 [81813.910006] Lustre: Skipped 3 previous similar messages [81814.109660] Lustre: server umount fir-OST0003 complete [81814.907543] Lustre: Failing over fir-OST000b [81814.929007] Lustre: server umount fir-OST0001 complete [81814.934150] Lustre: Skipped 1 previous similar message [81822.112359] md7: detected capacity change from 64011422924800 to 0 [81822.118583] md: md7 stopped. [81822.136777] md1: detected capacity change from 64011422924800 to 0 [81822.142977] md: md1 stopped. [81822.178115] md3: detected capacity change from 64011422924800 to 0 [81822.180025] md5: detected capacity change from 64011422924800 to 0 [81822.180034] md: md5 stopped. [81822.187002] md11: detected capacity change from 64011422924800 to 0 [81822.187009] md: md11 stopped. [81822.202609] md: md3 stopped. [81822.205705] md9: detected capacity change from 64011422924800 to 0 [81822.211893] md: md9 stopped. [81825.949104] LNet: Removed LNI 10.0.10.102@o2ib7 [81875.803880] md: md11 stopped. [81875.847578] md/raid:md11: device dm-50 operational as raid disk 0 [81875.853679] md/raid:md11: device dm-46 operational as raid disk 9 [81875.859774] md/raid:md11: device dm-30 operational as raid disk 8 [81875.865874] md/raid:md11: device dm-112 operational as raid disk 7 [81875.872055] md/raid:md11: device dm-114 operational as raid disk 6 [81875.878243] md/raid:md11: device dm-80 operational as raid disk 5 [81875.884344] md/raid:md11: device dm-48 operational as raid disk 4 [81875.890445] md/raid:md11: device dm-40 operational as raid disk 3 [81875.896536] md/raid:md11: device dm-104 operational as raid disk 2 [81875.902718] md/raid:md11: device dm-95 operational as raid disk 1 [81875.909686] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [81875.928193] md11: detected capacity change from 0 to 64011422924800 [81875.941719] md: md9 stopped. [81875.961889] md/raid:md9: device dm-47 operational as raid disk 0 [81875.967904] md/raid:md9: device dm-42 operational as raid disk 9 [81875.973916] md/raid:md9: device dm-39 operational as raid disk 8 [81875.979930] md/raid:md9: device dm-28 operational as raid disk 7 [81875.985935] md/raid:md9: device dm-116 operational as raid disk 6 [81875.992029] md/raid:md9: device dm-71 operational as raid disk 5 [81875.998035] md/raid:md9: device dm-110 operational as raid disk 4 [81876.004127] md/raid:md9: device dm-94 operational as raid disk 3 [81876.010136] md/raid:md9: device dm-33 operational as raid disk 2 [81876.016149] md/raid:md9: device dm-53 operational as raid disk 1 [81876.022875] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [81876.040431] md9: detected capacity change from 0 to 64011422924800 [81876.057695] md: md7 stopped. [81876.118814] md/raid:md7: device dm-92 operational as raid disk 0 [81876.124831] md/raid:md7: device dm-38 operational as raid disk 9 [81876.130849] md/raid:md7: device dm-49 operational as raid disk 8 [81876.136874] md/raid:md7: device dm-97 operational as raid disk 7 [81876.142896] md/raid:md7: device dm-111 operational as raid disk 6 [81876.149008] md/raid:md7: device dm-96 operational as raid disk 5 [81876.155028] md/raid:md7: device dm-101 operational as raid disk 4 [81876.161138] md/raid:md7: device dm-91 operational as raid disk 3 [81876.167162] md/raid:md7: device dm-86 operational as raid disk 2 [81876.173185] md/raid:md7: device dm-109 operational as raid disk 1 [81876.180269] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [81876.198936] md7: detected capacity change from 0 to 64011422924800 [81876.207860] md: md5 stopped. [81876.242287] md/raid:md5: device dm-84 operational as raid disk 0 [81876.248324] md/raid:md5: device dm-8 operational as raid disk 9 [81876.254272] md/raid:md5: device dm-3 operational as raid disk 8 [81876.260211] md/raid:md5: device dm-22 operational as raid disk 7 [81876.266248] md/raid:md5: device dm-21 operational as raid disk 6 [81876.272272] md/raid:md5: device dm-67 operational as raid disk 5 [81876.278289] md/raid:md5: device dm-36 operational as raid disk 4 [81876.284310] md/raid:md5: device dm-10 operational as raid disk 3 [81876.290328] md/raid:md5: device dm-76 operational as raid disk 2 [81876.296345] md/raid:md5: device dm-56 operational as raid disk 1 [81876.303412] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [81876.329826] md5: detected capacity change from 0 to 64011422924800 [81876.349483] md: md3 stopped. [81876.380505] md/raid:md3: device dm-2 operational as raid disk 0 [81876.386440] md/raid:md3: device dm-0 operational as raid disk 9 [81876.392369] md/raid:md3: device dm-65 operational as raid disk 8 [81876.398386] md/raid:md3: device dm-24 operational as raid disk 7 [81876.404404] md/raid:md3: device dm-14 operational as raid disk 6 [81876.410417] md/raid:md3: device dm-118 operational as raid disk 5 [81876.416525] md/raid:md3: device dm-64 operational as raid disk 4 [81876.422545] md/raid:md3: device dm-75 operational as raid disk 3 [81876.428562] md/raid:md3: device dm-31 operational as raid disk 2 [81876.434580] md/raid:md3: device dm-15 operational as raid disk 1 [81876.441716] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [81876.460408] md3: detected capacity change from 0 to 64011422924800 [81876.488794] md: md1 stopped. [81876.525512] md/raid:md1: device dm-26 operational as raid disk 0 [81876.531532] md/raid:md1: device dm-9 operational as raid disk 9 [81876.537462] md/raid:md1: device dm-16 operational as raid disk 8 [81876.543472] md/raid:md1: device dm-59 operational as raid disk 7 [81876.549486] md/raid:md1: device dm-23 operational as raid disk 6 [81876.555503] md/raid:md1: device dm-60 operational as raid disk 5 [81876.561517] md/raid:md1: device dm-34 operational as raid disk 4 [81876.567529] md/raid:md1: device dm-68 operational as raid disk 3 [81876.573544] md/raid:md1: device dm-43 operational as raid disk 2 [81876.579557] md/raid:md1: device dm-57 operational as raid disk 1 [81876.586251] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [81876.605026] md1: detected capacity change from 0 to 64011422924800 [81894.651060] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [81895.007981] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81895.044031] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [81895.363926] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81895.369063] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [81895.681057] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81895.972032] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [81896.295631] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81896.592073] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [81896.689071] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [81896.906537] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81897.010003] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81922.507804] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [81922.515395] alg: No test for adler32 (adler32-zlib) [81923.326961] Lustre: Lustre: Build Version: 2.12.0 [81923.466171] LNet: Using FastReg for registration [81923.481746] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [81950.832512] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [81951.100513] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [81951.159337] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81951.448350] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81951.551032] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [81951.566471] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [81951.844535] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [81951.875488] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [81951.905937] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81952.112462] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [81952.184841] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81952.292820] Lustre: fir-OST0001: new disk, initializing [81952.309565] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [81952.371874] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [81952.422743] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [81952.584490] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [81952.651903] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81952.702206] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81952.861492] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [81952.883129] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [81953.161861] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [81953.297519] Lustre: srv-fir-OST0001: No data found on store. Initialize space [81953.322520] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [81953.721516] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [81953.773522] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [81954.146198] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [81954.150249] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [81954.365200] Lustre: fir-OST0005: new disk, initializing [81954.370493] Lustre: srv-fir-OST0005: No data found on store. Initialize space [81954.386559] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [81955.405299] Lustre: fir-OST000b: new disk, initializing [81955.410589] Lustre: srv-fir-OST000b: No data found on store. Initialize space [81955.430832] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [81957.731761] Lustre: fir-OST0001: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [81957.731762] Lustre: fir-OST0005: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [81957.752100] Lustre: Skipped 4 previous similar messages [81959.083875] Lustre: fir-OST0001: Connection restored to fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) [81959.093101] Lustre: Skipped 3 previous similar messages [81959.097197] Lustre: cli-fir-OST0005-super: Allocated super-sequence [0x0000000300000400-0x0000000340000400]:5:ost] [81960.094443] Lustre: fir-OST0003: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [81960.104621] Lustre: Skipped 4 previous similar messages [81965.689167] Lustre: fir-OST0003: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [81965.699605] Lustre: Skipped 1 previous similar message [81972.025658] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [81972.042767] LustreError: Skipped 4 previous similar messages [81973.441231] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [81973.458599] LustreError: Skipped 3 previous similar messages [81985.497839] Lustre: fir-OST000b: Connection restored to fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) [81985.507066] Lustre: Skipped 3 previous similar messages [81985.528567] Lustre: cli-fir-OST0009-super: Allocated super-sequence [0x00000005c0000400-0x0000000600000400]:9:ost] [81985.538920] Lustre: Skipped 2 previous similar messages [81997.026279] Lustre: fir-OST0003: Connection restored to (at 10.9.0.2@o2ib4) [81997.033340] Lustre: Skipped 19 previous similar messages [82094.990555] Lustre: fir-OST0005: Connection restored to ac6fa15d-85d8-d7f7-0237-c13aeb11aa09 (at 10.0.10.3@o2ib7) [82095.000813] Lustre: Skipped 8 previous similar messages [82136.995028] Lustre: fir-OST0001: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [82137.005472] Lustre: Skipped 5 previous similar messages [82264.435952] Lustre: fir-OST0009: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [82467.503116] Lustre: fir-OST0009: Connection restored to ac6fa15d-85d8-d7f7-0237-c13aeb11aa09 (at 10.0.10.3@o2ib7) [82467.513404] Lustre: Skipped 2 previous similar messages [82755.284713] Lustre: fir-OST0005: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [82755.295164] Lustre: Skipped 5 previous similar messages [92357.374056] Lustre: fir-OST000b: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [92357.384496] Lustre: Skipped 2 previous similar messages [96741.992501] Lustre: fir-OST0009: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [97078.596082] Lustre: fir-OST0003: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [97868.338949] Lustre: fir-OST0007: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [98215.258736] Lustre: fir-OST0001: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [98571.571660] Lustre: fir-OST0005: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [98940.714616] Lustre: fir-OST000b: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [99706.669023] Lustre: fir-OST0009: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [197046.664977] md: data-check of RAID array md1 [197052.685277] md: data-check of RAID array md3 [197058.769857] md: data-check of RAID array md5 [197064.774233] md: data-check of RAID array md7 [197070.878786] md: data-check of RAID array md9 [197076.892093] md: data-check of RAID array md11 [224416.183995] perf: interrupt took too long (2501 > 2500), lowering kernel.perf_event_max_sample_rate to 79000 [237575.340282] perf: interrupt took too long (3127 > 3126), lowering kernel.perf_event_max_sample_rate to 63000 [307373.856741] md: md11: data-check done. [310457.992824] md: md5: data-check done. [402814.584292] Lustre: fir-OST0003: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [402845.215061] Lustre: fir-OST0009: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [402845.225587] Lustre: Skipped 3 previous similar messages [403073.662777] Lustre: fir-OST000b: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [411610.050165] Lustre: fir-OST0001: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [411610.060704] Lustre: Skipped 2 previous similar messages [416936.330937] Lustre: fir-OST0009: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [416936.341463] Lustre: Skipped 7 previous similar messages [417015.042338] LNet: 40731:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [417015.052547] Lustre: 41063:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548194763/real 1548194763] req@ffff90cb65f4a400 x1623048517166256/t0(0) o400->fir-MDT0003-lwp-OST0003@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194770 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [417015.052556] Lustre: fir-MDT0001-lwp-OST0007: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [417015.097469] Lustre: 41063:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [417016.042376] LNet: 40731:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 21 seconds [417016.052633] LNet: 40731:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [417016.062082] Lustre: 41066:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548194763/real 1548194764] req@ffff90cb65f4f200 x1623048517166272/t0(0) o400->fir-MDT0001-lwp-OST0003@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194770 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [417016.062090] Lustre: fir-MDT0003-lwp-OST0009: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [417016.062092] Lustre: Skipped 7 previous similar messages [417016.112290] Lustre: 41066:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [417017.042400] LNet: 40731:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 14 seconds [417017.052655] LNet: 40731:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [417017.115402] Lustre: 103022:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548194759/real 1548194759] req@ffff90b11b9b8600 x1623048517165952/t0(0) o39->fir-MDT0002-lwp-OST0001@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194765 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [417017.143886] Lustre: 103022:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [417023.116552] Lustre: 103020:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548194765/real 1548194765] req@ffff90a53b179500 x1623048517166544/t0(0) o39->fir-MDT0000-lwp-OST0005@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194771 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [417023.122012] Lustre: Failing over fir-OST000b [417023.149395] Lustre: 103020:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [417023.967845] Lustre: server umount fir-OST0009 complete [417023.973081] Lustre: Skipped 3 previous similar messages [417024.421308] md11: detected capacity change from 64011422924800 to 0 [417024.427677] md: md11 stopped. [417024.495353] md: md1: data-check interrupted. [417024.545577] md: md9: data-check interrupted. [417024.781587] md: md7: data-check interrupted. [417024.858624] md1: detected capacity change from 64011422924800 to 0 [417024.864908] md: md1 stopped. [417024.876209] md9: detected capacity change from 64011422924800 to 0 [417024.882493] md: md9 stopped. [417024.950775] md7: detected capacity change from 64011422924800 to 0 [417024.957079] md: md7 stopped. [417025.036066] md5: detected capacity change from 64011422924800 to 0 [417025.042348] md: md5 stopped. [417025.434599] md: md3: data-check interrupted. [417025.573962] md3: detected capacity change from 64011422924800 to 0 [417025.580250] md: md3 stopped. [417029.781719] LNet: Removed LNI 10.0.10.102@o2ib7 [417815.645575] md: md3 stopped. [417815.702296] md/raid:md3: device dm-2 operational as raid disk 0 [417815.708316] md/raid:md3: device dm-0 operational as raid disk 9 [417815.714333] md/raid:md3: device dm-65 operational as raid disk 8 [417815.720442] md/raid:md3: device dm-24 operational as raid disk 7 [417815.726545] md/raid:md3: device dm-14 operational as raid disk 6 [417815.732683] md/raid:md3: device dm-118 operational as raid disk 5 [417815.738873] md/raid:md3: device dm-64 operational as raid disk 4 [417815.744976] md/raid:md3: device dm-75 operational as raid disk 3 [417815.751080] md/raid:md3: device dm-31 operational as raid disk 2 [417815.757190] md/raid:md3: device dm-15 operational as raid disk 1 [417815.764213] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [417815.803204] md3: detected capacity change from 0 to 64011422924800 [417815.814036] md: md1 stopped. [417815.829111] md/raid:md1: device dm-26 operational as raid disk 0 [417815.835214] md/raid:md1: device dm-9 operational as raid disk 9 [417815.841226] md/raid:md1: device dm-16 operational as raid disk 8 [417815.847327] md/raid:md1: device dm-59 operational as raid disk 7 [417815.853428] md/raid:md1: device dm-23 operational as raid disk 6 [417815.859523] md/raid:md1: device dm-60 operational as raid disk 5 [417815.865621] md/raid:md1: device dm-34 operational as raid disk 4 [417815.871717] md/raid:md1: device dm-68 operational as raid disk 3 [417815.877817] md/raid:md1: device dm-43 operational as raid disk 2 [417815.883917] md/raid:md1: device dm-57 operational as raid disk 1 [417815.890757] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [417815.919774] md1: detected capacity change from 0 to 64011422924800 [417815.932251] md: md7 stopped. [417815.991515] md/raid:md7: device dm-92 operational as raid disk 0 [417815.997624] md/raid:md7: device dm-38 operational as raid disk 9 [417816.003734] md/raid:md7: device dm-49 operational as raid disk 8 [417816.009841] md/raid:md7: device dm-97 operational as raid disk 7 [417816.015950] md/raid:md7: device dm-111 operational as raid disk 6 [417816.022145] md/raid:md7: device dm-96 operational as raid disk 5 [417816.028246] md/raid:md7: device dm-101 operational as raid disk 4 [417816.034436] md/raid:md7: device dm-91 operational as raid disk 3 [417816.040544] md/raid:md7: device dm-86 operational as raid disk 2 [417816.046651] md/raid:md7: device dm-109 operational as raid disk 1 [417816.053768] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [417816.085298] md7: detected capacity change from 0 to 64011422924800 [417816.102393] md: md11 stopped. [417816.165862] md/raid:md11: device dm-50 operational as raid disk 0 [417816.172058] md/raid:md11: device dm-46 operational as raid disk 9 [417816.178256] md/raid:md11: device dm-30 operational as raid disk 8 [417816.184451] md/raid:md11: device dm-112 operational as raid disk 7 [417816.190730] md/raid:md11: device dm-114 operational as raid disk 6 [417816.197007] md/raid:md11: device dm-80 operational as raid disk 5 [417816.203194] md/raid:md11: device dm-48 operational as raid disk 4 [417816.209384] md/raid:md11: device dm-40 operational as raid disk 3 [417816.215565] md/raid:md11: device dm-104 operational as raid disk 2 [417816.221839] md/raid:md11: device dm-95 operational as raid disk 1 [417816.228813] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [417816.283903] md11: detected capacity change from 0 to 64011422924800 [417816.296457] md: md9 stopped. [417816.325043] md/raid:md9: device dm-47 operational as raid disk 0 [417816.331142] md/raid:md9: device dm-42 operational as raid disk 9 [417816.337256] md/raid:md9: device dm-39 operational as raid disk 8 [417816.343362] md/raid:md9: device dm-28 operational as raid disk 7 [417816.349472] md/raid:md9: device dm-116 operational as raid disk 6 [417816.355661] md/raid:md9: device dm-71 operational as raid disk 5 [417816.361759] md/raid:md9: device dm-110 operational as raid disk 4 [417816.367939] md/raid:md9: device dm-94 operational as raid disk 3 [417816.374040] md/raid:md9: device dm-33 operational as raid disk 2 [417816.380142] md/raid:md9: device dm-53 operational as raid disk 1 [417816.387026] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [417816.421289] md9: detected capacity change from 0 to 64011422924800 [417816.434265] md: md5 stopped. [417816.532796] md/raid:md5: device dm-84 operational as raid disk 0 [417816.538911] md/raid:md5: device dm-8 operational as raid disk 9 [417816.544943] md/raid:md5: device dm-3 operational as raid disk 8 [417816.550970] md/raid:md5: device dm-22 operational as raid disk 7 [417816.557079] md/raid:md5: device dm-21 operational as raid disk 6 [417816.563187] md/raid:md5: device dm-67 operational as raid disk 5 [417816.569307] md/raid:md5: device dm-36 operational as raid disk 4 [417816.575421] md/raid:md5: device dm-10 operational as raid disk 3 [417816.581526] md/raid:md5: device dm-76 operational as raid disk 2 [417816.587634] md/raid:md5: device dm-56 operational as raid disk 1 [417816.594541] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [417816.639998] md5: detected capacity change from 0 to 64011422924800 [417834.886480] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [417834.914482] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [417835.211509] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [417835.248489] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417835.254810] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417835.542378] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417836.462517] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [417836.583528] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [417836.918987] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417836.944582] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417839.126587] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [417839.497142] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417892.162545] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [417892.170288] alg: No test for adler32 (adler32-zlib) [417892.986116] Lustre: Lustre: Build Version: 2.12.0 [417893.117846] LNet: Using FastReg for registration [417893.133266] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [417896.786331] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [417897.092118] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [417897.124528] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417897.375105] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [417897.415364] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417897.674079] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [417897.681137] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [417897.714478] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417897.961090] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [417897.977426] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417898.025725] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417898.051111] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [417898.264655] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417898.330070] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [417898.395197] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417898.413086] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [417898.662558] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417898.694097] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [417898.713530] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417898.995968] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417899.221392] LustreError: 137-5: fir-OST000b_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [417899.245132] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [417899.255422] Lustre: fir-OST000b: new disk, initializing [417899.260110] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [417899.554482] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417899.559227] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417900.267169] Lustre: srv-fir-OST000b: No data found on store. Initialize space [417900.618944] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [417901.664960] Lustre: fir-OST0005: new disk, initializing [417901.670335] Lustre: srv-fir-OST0005: No data found on store. Initialize space [417901.690217] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [417905.947823] Lustre: fir-OST000b: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [417905.956530] Lustre: Skipped 2 previous similar messages [417905.959629] Lustre: cli-fir-OST0005-super: Allocated super-sequence [0x0000000380000400-0x00000003c0000400]:5:ost] [417907.483289] Lustre: fir-OST0005: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [417907.493561] Lustre: Skipped 7 previous similar messages [417932.483764] Lustre: fir-OST0001: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [417932.494034] Lustre: Skipped 21 previous similar messages [417969.843945] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [417969.861150] LustreError: Skipped 6 previous similar messages [417971.258739] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [417971.276203] LustreError: Skipped 5 previous similar messages [417994.844642] Lustre: fir-OST0003: Connection restored to (at 10.9.0.2@o2ib4) [417994.851795] Lustre: Skipped 5 previous similar messages [418002.928781] Lustre: fir-MDT0002-lwp-OST0005: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [418002.944857] Lustre: Skipped 19 previous similar messages [418007.568074] Lustre: Failing over fir-OST000b [418007.572507] Lustre: Skipped 4 previous similar messages [418007.643218] Lustre: fir-OST0001: Not available for connect from 10.8.9.6@o2ib6 (stopping) [418008.361270] Lustre: server umount fir-OST0005 complete [418008.366524] Lustre: Skipped 3 previous similar messages [418008.711244] md5: detected capacity change from 64011422924800 to 0 [418008.717527] md: md5 stopped. [418008.732380] md1: detected capacity change from 64011422924800 to 0 [418008.732555] md9: detected capacity change from 64011422924800 to 0 [418008.732564] md: md9 stopped. [418008.747892] md: md1 stopped. [418008.749222] md7: detected capacity change from 64011422924800 to 0 [418008.749228] md: md7 stopped. [418008.779797] md11: detected capacity change from 64011422924800 to 0 [418008.786159] md: md11 stopped. [418008.937814] md3: detected capacity change from 64011422924800 to 0 [418008.944100] md: md3 stopped. [418013.147888] LNet: Removed LNI 10.0.10.102@o2ib7 [418042.461687] md: md11 stopped. [418042.500152] md/raid:md11: device dm-50 operational as raid disk 0 [418042.506341] md/raid:md11: device dm-46 operational as raid disk 9 [418042.512528] md/raid:md11: device dm-30 operational as raid disk 8 [418042.518742] md/raid:md11: device dm-112 operational as raid disk 7 [418042.525016] md/raid:md11: device dm-114 operational as raid disk 6 [418042.531308] md/raid:md11: device dm-80 operational as raid disk 5 [418042.537487] md/raid:md11: device dm-48 operational as raid disk 4 [418042.543683] md/raid:md11: device dm-40 operational as raid disk 3 [418042.549863] md/raid:md11: device dm-104 operational as raid disk 2 [418042.556147] md/raid:md11: device dm-95 operational as raid disk 1 [418042.563147] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [418042.588248] md11: detected capacity change from 0 to 64011422924800 [418042.595600] md: md7 stopped. [418042.644452] md/raid:md7: device dm-92 operational as raid disk 0 [418042.650560] md/raid:md7: device dm-38 operational as raid disk 9 [418042.656665] md/raid:md7: device dm-49 operational as raid disk 8 [418042.662763] md/raid:md7: device dm-97 operational as raid disk 7 [418042.668869] md/raid:md7: device dm-111 operational as raid disk 6 [418042.675068] md/raid:md7: device dm-96 operational as raid disk 5 [418042.681176] md/raid:md7: device dm-101 operational as raid disk 4 [418042.687389] md/raid:md7: device dm-91 operational as raid disk 3 [418042.693492] md/raid:md7: device dm-86 operational as raid disk 2 [418042.699611] md/raid:md7: device dm-109 operational as raid disk 1 [418042.706543] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [418042.742997] md7: detected capacity change from 0 to 64011422924800 [418042.751545] md: md3 stopped. [418042.791502] md/raid:md3: device dm-2 operational as raid disk 0 [418042.797531] md/raid:md3: device dm-0 operational as raid disk 9 [418042.803570] md/raid:md3: device dm-65 operational as raid disk 8 [418042.809679] md/raid:md3: device dm-24 operational as raid disk 7 [418042.815831] md/raid:md3: device dm-14 operational as raid disk 6 [418042.821939] md/raid:md3: device dm-118 operational as raid disk 5 [418042.828129] md/raid:md3: device dm-64 operational as raid disk 4 [418042.834231] md/raid:md3: device dm-75 operational as raid disk 3 [418042.840333] md/raid:md3: device dm-31 operational as raid disk 2 [418042.846427] md/raid:md3: device dm-15 operational as raid disk 1 [418042.853528] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [418042.872553] md3: detected capacity change from 0 to 64011422924800 [418042.881939] md: md5 stopped. [418042.937201] md/raid:md5: device dm-84 operational as raid disk 0 [418042.943372] md/raid:md5: device dm-8 operational as raid disk 9 [418042.949392] md/raid:md5: device dm-3 operational as raid disk 8 [418042.955413] md/raid:md5: device dm-22 operational as raid disk 7 [418042.961521] md/raid:md5: device dm-21 operational as raid disk 6 [418042.967622] md/raid:md5: device dm-67 operational as raid disk 5 [418042.973718] md/raid:md5: device dm-36 operational as raid disk 4 [418042.979812] md/raid:md5: device dm-10 operational as raid disk 3 [418042.985918] md/raid:md5: device dm-76 operational as raid disk 2 [418042.992021] md/raid:md5: device dm-56 operational as raid disk 1 [418042.998892] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [418043.017873] md5: detected capacity change from 0 to 64011422924800 [418043.032071] md: md9 stopped. [418043.106308] md/raid:md9: device dm-47 operational as raid disk 0 [418043.112408] md/raid:md9: device dm-42 operational as raid disk 9 [418043.118520] md/raid:md9: device dm-39 operational as raid disk 8 [418043.124629] md/raid:md9: device dm-28 operational as raid disk 7 [418043.130740] md/raid:md9: device dm-116 operational as raid disk 6 [418043.136941] md/raid:md9: device dm-71 operational as raid disk 5 [418043.143042] md/raid:md9: device dm-110 operational as raid disk 4 [418043.149232] md/raid:md9: device dm-94 operational as raid disk 3 [418043.155345] md/raid:md9: device dm-33 operational as raid disk 2 [418043.161457] md/raid:md9: device dm-53 operational as raid disk 1 [418043.168705] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [418043.186828] md9: detected capacity change from 0 to 64011422924800 [418043.194796] md: md1 stopped. [418043.225436] md/raid:md1: device dm-26 operational as raid disk 0 [418043.231535] md/raid:md1: device dm-9 operational as raid disk 9 [418043.237555] md/raid:md1: device dm-16 operational as raid disk 8 [418043.243654] md/raid:md1: device dm-59 operational as raid disk 7 [418043.249755] md/raid:md1: device dm-23 operational as raid disk 6 [418043.255863] md/raid:md1: device dm-60 operational as raid disk 5 [418043.261966] md/raid:md1: device dm-34 operational as raid disk 4 [418043.268069] md/raid:md1: device dm-68 operational as raid disk 3 [418043.274174] md/raid:md1: device dm-43 operational as raid disk 2 [418043.280278] md/raid:md1: device dm-57 operational as raid disk 1 [418043.287526] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [418043.324587] md1: detected capacity change from 0 to 64011422924800 [418062.434311] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [418062.519319] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [418062.758125] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418062.846188] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418062.861329] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [418063.192416] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418064.301340] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [418064.636767] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418067.772441] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [418068.098081] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418068.901475] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [418069.227583] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418102.930914] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [418102.938637] alg: No test for adler32 (adler32-zlib) [418103.743314] Lustre: Lustre: Build Version: 2.12.0 [418103.871948] LNet: Using FastReg for registration [418103.887750] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [418107.564484] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [418107.835530] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [418107.911797] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418108.144518] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [418108.161970] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418108.298468] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [418108.477430] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418108.599763] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418108.662093] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [418108.737542] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [418108.771489] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [418108.938491] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [418109.004867] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418109.068032] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [418109.083613] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418109.085009] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418109.239412] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418109.388445] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418109.655493] LDISKFS-fs (md11): file extents enabled [418109.657496] LDISKFS-fs (md1): file extents enabled [418109.657497] , maximum tree depth=5 [418109.667108] , maximum tree depth=5 [418109.759500] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [418109.958428] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418109.966417] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418110.018455] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [418110.035832] LustreError: Skipped 1 previous similar message [418110.049042] Lustre: fir-OST0005: new disk, initializing [418110.062286] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418111.054528] Lustre: srv-fir-OST0005: No data found on store. Initialize space [418111.076637] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [418112.164304] Lustre: fir-OST0009: new disk, initializing [418112.169683] Lustre: srv-fir-OST0009: No data found on store. Initialize space [418112.187488] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [418113.346599] Lustre: fir-OST0003: new disk, initializing [418113.351940] Lustre: Skipped 3 previous similar messages [418113.357356] Lustre: srv-fir-OST0003: No data found on store. Initialize space [418113.364582] Lustre: Skipped 3 previous similar messages [418113.379734] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [418113.388007] Lustre: Skipped 3 previous similar messages [418113.888764] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [418113.905966] LustreError: Skipped 5 previous similar messages [418114.325265] Lustre: fir-OST0005: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [418114.334570] Lustre: Skipped 1 previous similar message [418114.390764] Lustre: cli-fir-OST0005-super: Allocated super-sequence [0x0000000480000400-0x00000004c0000400]:5:ost] [418115.329995] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [418115.347453] LustreError: Skipped 3 previous similar messages [418118.758957] Lustre: fir-OST0005: Connection restored to (at 10.0.10.52@o2ib7) [418118.766289] Lustre: Skipped 10 previous similar messages [418133.711910] Lustre: fir-OST000b: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [418133.711911] Lustre: fir-OST0003: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [418133.711914] Lustre: Skipped 13 previous similar messages [418138.889386] Lustre: fir-OST0003: Connection restored to (at 10.9.0.2@o2ib4) [418138.896529] Lustre: Skipped 10 previous similar messages [418428.802911] Lustre: fir-OST0003: Connection restored to 3d981e2a-06a8-218c-1aa5-680f19d3d6b1 (at 10.0.10.3@o2ib7) [418428.802912] Lustre: fir-OST0001: Connection restored to 3d981e2a-06a8-218c-1aa5-680f19d3d6b1 (at 10.0.10.3@o2ib7) [418428.802914] Lustre: Skipped 7 previous similar messages [418428.828907] Lustre: Skipped 3 previous similar messages [418489.340632] Lustre: fir-OST0001: Connection restored to bd488cd4-324a-aaa3-2569-ed85c1419cd4 (at 10.9.101.59@o2ib4) [418489.351160] Lustre: Skipped 3 previous similar messages [418985.117773] Lustre: 2148:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548196727/real 1548196727] req@ffff90ab2e9c8300 x1623401013726480/t0(0) o39->fir-MDT0002-lwp-OST0005@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196733 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [418985.146069] Lustre: 2148:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [418991.121920] Lustre: 2153:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548196733/real 1548196733] req@ffff90b76ddb0300 x1623401013726608/t0(0) o39->fir-MDT0000-lwp-OST0009@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196739 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [418991.150238] Lustre: 2153:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [418992.893974] LNet: 121829:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 4 seconds [418992.904282] Lustre: 121901:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548196740/real 1548196741] req@ffff90db8a95c200 x1623401013726816/t0(0) o400->fir-MDT0001-lwp-OST0007@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196747 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [418992.904292] Lustre: fir-MDT0001-lwp-OST0005: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [418992.949245] Lustre: 121901:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [418993.894003] LNet: 121829:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 22 seconds [418993.904342] LNet: 121829:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [418993.915712] Lustre: Failing over fir-OST0001 [418993.920075] Lustre: Skipped 2 previous similar messages [418994.692014] Lustre: server umount fir-OST0005 complete [418994.895988] Lustre: Failing over fir-OST000b [418994.900353] Lustre: Skipped 1 previous similar message [418995.524314] md5: detected capacity change from 64011422924800 to 0 [418995.530611] md: md5 stopped. [418995.541022] md9: detected capacity change from 64011422924800 to 0 [418995.547305] md: md9 stopped. [418995.571897] md1: detected capacity change from 64011422924800 to 0 [418995.578171] md: md1 stopped. [418995.894531] Lustre: server umount fir-OST0003 complete [418995.899769] Lustre: Skipped 2 previous similar messages [418996.167636] md11: detected capacity change from 64011422924800 to 0 [418996.174013] md: md11 stopped. [418996.205314] md3: detected capacity change from 64011422924800 to 0 [418996.211601] md: md3 stopped. [418996.894099] LNet: 121829:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [418996.904358] LNet: 121829:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 9 previous similar messages [418996.913875] Lustre: 2151:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548196739/real 1548196745] req@ffff90b92aaf0000 x1623401013726672/t0(0) o39->fir-MDT0003-lwp-OST0007@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196745 ref 2 fl Rpc:eXN/0/ffffffff rc 0/-1 [418996.942555] Lustre: 2151:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [418996.955960] Lustre: Failing over fir-OST0007 [418997.836542] Lustre: server umount fir-OST0007 complete [418997.841773] Lustre: Skipped 1 previous similar message [418998.066909] md7: detected capacity change from 64011422924800 to 0 [418998.073190] md: md7 stopped. [419001.921195] LNet: Removed LNI 10.0.10.102@o2ib7 [419018.723935] md: md5 stopped. [419018.766840] md/raid:md5: device dm-84 operational as raid disk 0 [419018.772942] md/raid:md5: device dm-8 operational as raid disk 9 [419018.778947] md/raid:md5: device dm-3 operational as raid disk 8 [419018.784955] md/raid:md5: device dm-22 operational as raid disk 7 [419018.791055] md/raid:md5: device dm-21 operational as raid disk 6 [419018.797148] md/raid:md5: device dm-67 operational as raid disk 5 [419018.803242] md/raid:md5: device dm-36 operational as raid disk 4 [419018.809344] md/raid:md5: device dm-10 operational as raid disk 3 [419018.815446] md/raid:md5: device dm-76 operational as raid disk 2 [419018.821545] md/raid:md5: device dm-56 operational as raid disk 1 [419018.828485] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [419018.860119] md5: detected capacity change from 0 to 64011422924800 [419018.874042] md: md11 stopped. [419018.896076] md/raid:md11: device dm-50 operational as raid disk 0 [419018.902262] md/raid:md11: device dm-46 operational as raid disk 9 [419018.908450] md/raid:md11: device dm-30 operational as raid disk 8 [419018.914637] md/raid:md11: device dm-112 operational as raid disk 7 [419018.920918] md/raid:md11: device dm-114 operational as raid disk 6 [419018.927188] md/raid:md11: device dm-80 operational as raid disk 5 [419018.933376] md/raid:md11: device dm-48 operational as raid disk 4 [419018.939578] md/raid:md11: device dm-40 operational as raid disk 3 [419018.945760] md/raid:md11: device dm-104 operational as raid disk 2 [419018.952025] md/raid:md11: device dm-95 operational as raid disk 1 [419018.959028] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [419018.977656] md11: detected capacity change from 0 to 64011422924800 [419018.989519] md: md3 stopped. [419019.028282] md/raid:md3: device dm-2 operational as raid disk 0 [419019.034327] md/raid:md3: device dm-0 operational as raid disk 9 [419019.040350] md/raid:md3: device dm-65 operational as raid disk 8 [419019.046457] md/raid:md3: device dm-24 operational as raid disk 7 [419019.052559] md/raid:md3: device dm-14 operational as raid disk 6 [419019.058658] md/raid:md3: device dm-118 operational as raid disk 5 [419019.064848] md/raid:md3: device dm-64 operational as raid disk 4 [419019.070949] md/raid:md3: device dm-75 operational as raid disk 3 [419019.077050] md/raid:md3: device dm-31 operational as raid disk 2 [419019.083149] md/raid:md3: device dm-15 operational as raid disk 1 [419019.090025] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [419019.121180] md3: detected capacity change from 0 to 64011422924800 [419019.129980] md: md9 stopped. [419019.158167] md/raid:md9: device dm-47 operational as raid disk 0 [419019.164270] md/raid:md9: device dm-42 operational as raid disk 9 [419019.170366] md/raid:md9: device dm-39 operational as raid disk 8 [419019.176470] md/raid:md9: device dm-28 operational as raid disk 7 [419019.182566] md/raid:md9: device dm-116 operational as raid disk 6 [419019.188764] md/raid:md9: device dm-71 operational as raid disk 5 [419019.194872] md/raid:md9: device dm-110 operational as raid disk 4 [419019.201059] md/raid:md9: device dm-94 operational as raid disk 3 [419019.207160] md/raid:md9: device dm-33 operational as raid disk 2 [419019.213266] md/raid:md9: device dm-53 operational as raid disk 1 [419019.220185] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [419019.239255] md9: detected capacity change from 0 to 64011422924800 [419019.255005] md: md7 stopped. [419019.289686] md/raid:md7: device dm-92 operational as raid disk 0 [419019.295796] md/raid:md7: device dm-38 operational as raid disk 9 [419019.301901] md/raid:md7: device dm-49 operational as raid disk 8 [419019.308014] md/raid:md7: device dm-97 operational as raid disk 7 [419019.314124] md/raid:md7: device dm-111 operational as raid disk 6 [419019.320321] md/raid:md7: device dm-96 operational as raid disk 5 [419019.326428] md/raid:md7: device dm-101 operational as raid disk 4 [419019.332635] md/raid:md7: device dm-91 operational as raid disk 3 [419019.338741] md/raid:md7: device dm-86 operational as raid disk 2 [419019.344840] md/raid:md7: device dm-109 operational as raid disk 1 [419019.352129] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [419019.370971] md7: detected capacity change from 0 to 64011422924800 [419019.392393] md: md1 stopped. [419019.427021] md/raid:md1: device dm-26 operational as raid disk 0 [419019.433125] md/raid:md1: device dm-9 operational as raid disk 9 [419019.439140] md/raid:md1: device dm-16 operational as raid disk 8 [419019.445243] md/raid:md1: device dm-59 operational as raid disk 7 [419019.451353] md/raid:md1: device dm-23 operational as raid disk 6 [419019.457460] md/raid:md1: device dm-60 operational as raid disk 5 [419019.463562] md/raid:md1: device dm-34 operational as raid disk 4 [419019.469664] md/raid:md1: device dm-68 operational as raid disk 3 [419019.475772] md/raid:md1: device dm-43 operational as raid disk 2 [419019.481876] md/raid:md1: device dm-57 operational as raid disk 1 [419019.488942] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [419019.508096] md1: detected capacity change from 0 to 64011422924800 [419038.264290] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [419038.547300] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [419038.615207] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419038.780298] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [419038.875931] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419039.114286] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419039.187305] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [419039.512334] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419040.441378] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [419040.763002] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419043.506408] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [419043.851117] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419130.593268] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [419130.600992] alg: No test for adler32 (adler32-zlib) [419131.404598] Lustre: Lustre: Build Version: 2.12.0 [419131.526666] LNet: Using FastReg for registration [419131.542555] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [419135.005818] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [419135.286731] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [419135.341132] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419135.643855] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419135.663858] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [419135.787774] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [419136.000733] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419136.090365] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419136.162636] LDISKFS-fs (md11): file extents enabled [419136.164856] LDISKFS-fs (md5): file extents enabled [419136.164857] , maximum tree depth=5 [419136.174246] , maximum tree depth=5 [419136.455796] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [419136.479800] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [419136.501317] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419136.761847] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419136.763246] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [419136.786220] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419136.827548] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419137.097506] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419137.100803] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [419137.399342] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419137.468820] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [419137.741823] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [419137.771082] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419137.820293] Lustre: fir-OST0001: new disk, initializing [419138.042710] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419138.824797] Lustre: srv-fir-OST0001: No data found on store. Initialize space [419138.852308] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [419139.887636] Lustre: fir-OST0003: new disk, initializing [419139.893014] Lustre: srv-fir-OST0003: No data found on store. Initialize space [419139.914376] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [419141.128072] Lustre: fir-OST0009: new disk, initializing [419141.133392] Lustre: Skipped 3 previous similar messages [419141.138776] Lustre: srv-fir-OST0009: No data found on store. Initialize space [419141.146028] Lustre: Skipped 3 previous similar messages [419141.161823] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [419141.170089] Lustre: Skipped 3 previous similar messages [419141.721197] Lustre: fir-OST0001: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [419141.749897] Lustre: cli-fir-OST0003-super: Allocated super-sequence [0x0000000540000400-0x0000000580000400]:3:ost] [419141.760331] Lustre: Skipped 1 previous similar message [419142.656564] Lustre: fir-OST0001: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [419142.665875] Lustre: Skipped 12 previous similar messages [419147.848381] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [419147.865602] LustreError: Skipped 5 previous similar messages [419153.026253] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [419153.043453] LustreError: Skipped 10 previous similar messages [419154.468006] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [419154.485480] LustreError: Skipped 5 previous similar messages [419155.920812] Lustre: fir-OST0001: Connection restored to 31582059-27cb-3e3b-d317-11d93512a873 (at 10.0.10.3@o2ib7) [419155.920813] Lustre: fir-OST0003: Connection restored to 31582059-27cb-3e3b-d317-11d93512a873 (at 10.0.10.3@o2ib7) [419155.920815] Lustre: Skipped 10 previous similar messages [419155.946893] Lustre: Skipped 4 previous similar messages [419172.849095] Lustre: fir-OST0007: Connection restored to (at 10.8.9.6@o2ib6) [419172.849096] Lustre: fir-OST0005: Connection restored to (at 10.8.9.6@o2ib6) [419172.849097] Lustre: fir-OST0001: Connection restored to (at 10.8.9.6@o2ib6) [419172.849098] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [419172.877644] Lustre: Skipped 2 previous similar messages [419209.073858] Lustre: fir-OST0001: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [419209.084386] Lustre: Skipped 22 previous similar messages [420487.331139] Lustre: fir-OST0003: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [420487.341663] Lustre: Skipped 1 previous similar message [423022.342134] Lustre: fir-OST0009: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [423022.352504] Lustre: Skipped 9 previous similar messages [423266.637768] Lustre: fir-OST0007: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [424642.329873] Lustre: fir-OST0005: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [426243.471654] Lustre: fir-OST0005: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [426243.482006] Lustre: Skipped 4 previous similar messages [426245.930371] Lustre: fir-OST0003: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [426245.940723] Lustre: Skipped 1 previous similar message [426552.274374] Lustre: fir-OST0001: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [428363.688031] Lustre: fir-OST0005: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [428363.698383] Lustre: Skipped 6 previous similar messages [428519.948222] Lustre: fir-OST0003: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [428519.958581] Lustre: Skipped 5 previous similar messages [429999.041606] Lustre: fir-OST0007: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [430621.122036] Lustre: fir-OST0005: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [430621.890632] Lustre: fir-OST0009: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [430621.900981] Lustre: Skipped 1 previous similar message [434747.070961] Lustre: fir-OST000b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [480372.692406] Lustre: fir-OST0007: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [480372.702757] Lustre: Skipped 2 previous similar messages [480513.843314] Lustre: fir-OST0001: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [480513.853664] Lustre: Skipped 3 previous similar messages [491371.310740] Lustre: fir-OST0001: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [491371.310741] Lustre: fir-OST0005: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [491371.310743] Lustre: Skipped 4 previous similar messages [491371.337116] Lustre: Skipped 2 previous similar messages [491791.909943] Lustre: fir-OST0007: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [491791.909944] Lustre: fir-OST0003: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [491791.931026] Lustre: Skipped 3 previous similar messages [493394.060783] Lustre: fir-OST0007: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [493469.939234] Lustre: fir-OST0005: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [493704.214466] Lustre: fir-OST000b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [493981.085392] Lustre: fir-OST0009: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494002.342435] Lustre: fir-OST0001: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494034.754105] Lustre: fir-OST0003: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494127.696604] Lustre: fir-OST0007: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [494578.679779] Lustre: fir-OST000b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [494884.289544] Lustre: fir-OST0007: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494884.299895] Lustre: Skipped 2 previous similar messages [495051.525194] Lustre: fir-OST0003: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [495051.535715] Lustre: Skipped 1 previous similar message [500409.790476] Lustre: fir-OST000b: haven't heard from client 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90cb471e9800, cur 1548278156 expire 1548278006 last 1548277929 [500413.371920] Lustre: fir-OST0001: haven't heard from client 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) in 231 seconds. I think it's dead, and I am evicting it. exp ffff90babf98d000, cur 1548278160 expire 1548278010 last 1548277929 [500413.393541] Lustre: Skipped 1 previous similar message [504703.318798] Lustre: fir-OST0007: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [504703.329318] Lustre: Skipped 12 previous similar messages [535250.550409] Lustre: fir-OST0007: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [542167.611028] Lustre: fir-OST000b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [542184.171375] Lustre: fir-OST0001: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [542184.181724] Lustre: Skipped 1 previous similar message [570894.332792] Lustre: fir-OST0003: haven't heard from client 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) in 230 seconds. I think it's dead, and I am evicting it. exp ffff90babf96dc00, cur 1548348639 expire 1548348489 last 1548348409 [570894.354547] Lustre: Skipped 3 previous similar messages [578739.206707] Lustre: fir-OST0007: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [578739.217227] Lustre: Skipped 1 previous similar message [578921.718989] Lustre: fir-OST000b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [580084.513548] Lustre: fir-OST0005: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [580084.524079] Lustre: Skipped 2 previous similar messages [588455.569923] Lustre: fir-OST0009: Connection restored to 87096505-c370-0b84-8d41-ce56748694c4 (at 10.9.0.1@o2ib4) [588455.569925] Lustre: fir-OST0005: Connection restored to 87096505-c370-0b84-8d41-ce56748694c4 (at 10.9.0.1@o2ib4) [588455.569926] Lustre: fir-OST0001: Connection restored to 87096505-c370-0b84-8d41-ce56748694c4 (at 10.9.0.1@o2ib4) [588455.569929] Lustre: Skipped 4 previous similar messages [588455.569929] Lustre: Skipped 4 previous similar messages [588943.821944] Lustre: fir-OST0009: Connection restored to 31582059-27cb-3e3b-d317-11d93512a873 (at 10.0.10.3@o2ib7) [589417.521025] Lustre: 63256:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548367155/real 1548367155] req@ffff90a873cf8300 x1623402094364912/t0(0) o39->fir-MDT0002-lwp-OST000b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548367161 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [589417.549402] Lustre: 63256:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [589417.884028] LNet: 6503:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 4 seconds [589417.894157] Lustre: fir-MDT0001-lwp-OST0005: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [589417.910948] Lustre: Skipped 7 previous similar messages [589418.884066] LNet: 6503:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 21 seconds [589418.894238] LNet: 6503:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [589418.903596] Lustre: 6577:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548367162/real 1548367163] req@ffff90cb1b84b300 x1623402094365216/t0(0) o400->fir-MDT0003-lwp-OST0005@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548367169 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [589418.903604] Lustre: fir-MDT0001-lwp-OST000b: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [589418.948407] Lustre: 6577:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 11 previous similar messages [589420.884119] LNet: 6503:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 13 seconds [589420.894289] LNet: 6503:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [589423.522176] Lustre: 63255:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548367161/real 1548367161] req@ffff90cda96f0900 x1623402094364992/t0(0) o39->fir-MDT0000-lwp-OST0005@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548367167 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [589423.530643] Lustre: Failing over fir-OST0001 [589423.530644] Lustre: Failing over fir-OST0003 [589423.559297] Lustre: 63255:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [589424.371869] Lustre: server umount fir-OST0007 complete [589424.377107] Lustre: Skipped 1 previous similar message [589424.924787] md1: detected capacity change from 64011422924800 to 0 [589424.931066] md: md1 stopped. [589424.994207] md3: detected capacity change from 64011422924800 to 0 [589425.000490] md: md3 stopped. [589425.006439] Lustre: server umount fir-OST0005 complete [589425.011668] Lustre: Skipped 3 previous similar messages [589425.362575] md7: detected capacity change from 64011422924800 to 0 [589425.368862] md: md7 stopped. [589425.372716] md5: detected capacity change from 64011422924800 to 0 [589425.378993] md: md5 stopped. [589425.437727] md9: detected capacity change from 64011422924800 to 0 [589425.444014] md: md9 stopped. [589425.598232] md11: detected capacity change from 64011422924800 to 0 [589425.604598] md: md11 stopped. [589429.008314] LNet: Removed LNI 10.0.10.102@o2ib7 [589504.033292] md: md11 stopped. [589504.054523] md/raid:md11: device dm-50 operational as raid disk 0 [589504.060710] md/raid:md11: device dm-46 operational as raid disk 9 [589504.066896] md/raid:md11: device dm-30 operational as raid disk 8 [589504.073077] md/raid:md11: device dm-112 operational as raid disk 7 [589504.079367] md/raid:md11: device dm-114 operational as raid disk 6 [589504.085650] md/raid:md11: device dm-80 operational as raid disk 5 [589504.091848] md/raid:md11: device dm-48 operational as raid disk 4 [589504.098028] md/raid:md11: device dm-40 operational as raid disk 3 [589504.104205] md/raid:md11: device dm-104 operational as raid disk 2 [589504.110473] md/raid:md11: device dm-95 operational as raid disk 1 [589504.117526] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [589504.136211] md11: detected capacity change from 0 to 64011422924800 [589504.148282] md: md7 stopped. [589504.197644] md/raid:md7: device dm-92 operational as raid disk 0 [589504.203747] md/raid:md7: device dm-38 operational as raid disk 9 [589504.209852] md/raid:md7: device dm-49 operational as raid disk 8 [589504.215950] md/raid:md7: device dm-97 operational as raid disk 7 [589504.222044] md/raid:md7: device dm-111 operational as raid disk 6 [589504.228233] md/raid:md7: device dm-96 operational as raid disk 5 [589504.234330] md/raid:md7: device dm-101 operational as raid disk 4 [589504.240510] md/raid:md7: device dm-91 operational as raid disk 3 [589504.246605] md/raid:md7: device dm-86 operational as raid disk 2 [589504.252696] md/raid:md7: device dm-109 operational as raid disk 1 [589504.259644] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [589504.278378] md7: detected capacity change from 0 to 64011422924800 [589504.290518] md: md5 stopped. [589504.320299] md/raid:md5: device dm-84 operational as raid disk 0 [589504.326476] md/raid:md5: device dm-8 operational as raid disk 9 [589504.332503] md/raid:md5: device dm-3 operational as raid disk 8 [589504.338532] md/raid:md5: device dm-22 operational as raid disk 7 [589504.344636] md/raid:md5: device dm-21 operational as raid disk 6 [589504.350740] md/raid:md5: device dm-67 operational as raid disk 5 [589504.356846] md/raid:md5: device dm-36 operational as raid disk 4 [589504.362945] md/raid:md5: device dm-10 operational as raid disk 3 [589504.369046] md/raid:md5: device dm-76 operational as raid disk 2 [589504.375146] md/raid:md5: device dm-56 operational as raid disk 1 [589504.382251] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [589504.401381] md5: detected capacity change from 0 to 64011422924800 [589504.411907] md: md1 stopped. [589504.490941] md/raid:md1: device dm-26 operational as raid disk 0 [589504.497061] md/raid:md1: device dm-9 operational as raid disk 9 [589504.503083] md/raid:md1: device dm-16 operational as raid disk 8 [589504.509190] md/raid:md1: device dm-59 operational as raid disk 7 [589504.515290] md/raid:md1: device dm-23 operational as raid disk 6 [589504.521400] md/raid:md1: device dm-60 operational as raid disk 5 [589504.527509] md/raid:md1: device dm-34 operational as raid disk 4 [589504.533613] md/raid:md1: device dm-68 operational as raid disk 3 [589504.539715] md/raid:md1: device dm-43 operational as raid disk 2 [589504.545819] md/raid:md1: device dm-57 operational as raid disk 1 [589504.552724] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [589504.570393] md1: detected capacity change from 0 to 64011422924800 [589504.584735] md: md9 stopped. [589504.610080] md/raid:md9: device dm-47 operational as raid disk 0 [589504.616191] md/raid:md9: device dm-42 operational as raid disk 9 [589504.622299] md/raid:md9: device dm-39 operational as raid disk 8 [589504.628406] md/raid:md9: device dm-28 operational as raid disk 7 [589504.634504] md/raid:md9: device dm-116 operational as raid disk 6 [589504.640695] md/raid:md9: device dm-71 operational as raid disk 5 [589504.646797] md/raid:md9: device dm-110 operational as raid disk 4 [589504.652984] md/raid:md9: device dm-94 operational as raid disk 3 [589504.659084] md/raid:md9: device dm-33 operational as raid disk 2 [589504.665187] md/raid:md9: device dm-53 operational as raid disk 1 [589504.672076] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [589504.689976] md9: detected capacity change from 0 to 64011422924800 [589504.755243] md: md3 stopped. [589504.827896] md/raid:md3: device dm-2 operational as raid disk 0 [589504.833933] md/raid:md3: device dm-0 operational as raid disk 9 [589504.839964] md/raid:md3: device dm-65 operational as raid disk 8 [589504.846080] md/raid:md3: device dm-24 operational as raid disk 7 [589504.852178] md/raid:md3: device dm-14 operational as raid disk 6 [589504.858301] md/raid:md3: device dm-118 operational as raid disk 5 [589504.864488] md/raid:md3: device dm-64 operational as raid disk 4 [589504.870619] md/raid:md3: device dm-75 operational as raid disk 3 [589504.876725] md/raid:md3: device dm-31 operational as raid disk 2 [589504.882826] md/raid:md3: device dm-15 operational as raid disk 1 [589504.889830] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [589504.921267] md3: detected capacity change from 0 to 64011422924800 [589523.941976] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [589524.264563] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589524.269915] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [589524.475904] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [589524.611469] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589524.803528] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589525.350956] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [589525.688460] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589525.691949] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [589526.014025] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589526.351959] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [589526.682508] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589596.137373] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [589596.145149] alg: No test for adler32 (adler32-zlib) [589596.948410] Lustre: Lustre: Build Version: 2.12.0 [589597.074317] LNet: Using FastReg for registration [589597.090183] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [589600.658187] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [589600.986292] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589601.050953] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [589601.346953] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [589601.389817] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589601.632962] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [589601.675414] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589601.677891] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [589601.782882] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [589601.977513] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589601.980250] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589602.054959] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [589602.090399] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589602.337891] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [589602.390610] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589602.521731] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [589602.640280] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589602.861796] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589602.924890] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [589603.189920] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [589603.209883] Lustre: fir-OST000b: new disk, initializing [589603.226527] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589604.214950] Lustre: srv-fir-OST000b: No data found on store. Initialize space [589604.396204] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [589604.437544] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589604.666966] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [589604.965340] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589605.393691] Lustre: fir-OST0009: new disk, initializing [589605.399070] Lustre: srv-fir-OST0009: No data found on store. Initialize space [589605.414436] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [589605.814909] Lustre: fir-OST000b: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [589605.824214] Lustre: Skipped 1 previous similar message [589605.837804] Lustre: cli-fir-OST000b-super: Allocated super-sequence [0x0000000380000400-0x00000003c0000400]:b:ost] [589605.848239] Lustre: Skipped 1 previous similar message [589607.096828] Lustre: fir-OST000b: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [589607.106143] Lustre: Skipped 4 previous similar messages [589607.124344] Lustre: cli-fir-OST0007-super: Allocated super-sequence [0x0000000b00000400-0x0000000b40000400]:7:ost] [589607.134781] Lustre: Skipped 3 previous similar messages [589612.663145] Lustre: fir-OST0007: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [589612.672524] Lustre: Skipped 12 previous similar messages [589613.642248] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [589613.659444] LustreError: Skipped 5 previous similar messages [589622.172291] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [589622.189750] LustreError: Skipped 5 previous similar messages [589638.642965] Lustre: fir-OST0009: Connection restored to (at 10.9.0.2@o2ib4) [589638.650119] Lustre: Skipped 8 previous similar messages [589647.172801] Lustre: fir-OST0001: Connection restored to (at 10.9.101.60@o2ib4) [589647.180206] Lustre: Skipped 5 previous similar messages [589700.081192] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [589700.091546] Lustre: Skipped 5 previous similar messages [589818.478921] Lustre: fir-OST0001: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [589818.489446] Lustre: Skipped 5 previous similar messages [590119.346399] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [592501.364638] Lustre: fir-OST0001: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [592501.375174] Lustre: Skipped 10 previous similar messages [592680.650902] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [592680.650904] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [592680.671419] Lustre: Skipped 4 previous similar messages [592781.668753] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [599504.302393] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [599504.312745] Lustre: Skipped 5 previous similar messages [599700.695245] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [599700.705600] Lustre: Skipped 3 previous similar messages [600863.838589] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [600863.848941] Lustre: Skipped 3 previous similar messages [601154.198463] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [601154.208722] Lustre: Skipped 5 previous similar messages [601161.337150] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [601161.347414] Lustre: Skipped 2 previous similar messages [601182.978686] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [601182.988953] Lustre: Skipped 6 previous similar messages [601455.666793] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [601455.677157] Lustre: Skipped 1 previous similar message [603983.667720] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [603983.677986] Lustre: Skipped 5 previous similar messages [604797.889380] Lustre: fir-OST0001: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [604798.683901] Lustre: fir-OST0003: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [604811.015102] Lustre: fir-OST0009: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [605140.890528] Lustre: fir-OST0003: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [605140.901046] Lustre: Skipped 1 previous similar message [613759.499203] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [613759.509473] Lustre: Skipped 1 previous similar message [615564.221386] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [615864.231338] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [615864.241690] Lustre: Skipped 5 previous similar messages [616164.239043] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [616164.249397] Lustre: Skipped 5 previous similar messages [616464.247372] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [616464.257717] Lustre: Skipped 5 previous similar messages [616764.255118] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [616764.265472] Lustre: Skipped 5 previous similar messages [617064.263467] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [617064.273812] Lustre: Skipped 5 previous similar messages [617364.273882] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [617364.284228] Lustre: Skipped 5 previous similar messages [617664.279049] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [617664.289390] Lustre: Skipped 5 previous similar messages [617964.270808] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [617964.281166] Lustre: Skipped 5 previous similar messages [618264.295098] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [618264.305452] Lustre: Skipped 5 previous similar messages [618565.286810] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [618565.297166] Lustre: Skipped 5 previous similar messages [619165.304112] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [619165.314462] Lustre: Skipped 11 previous similar messages [619765.334334] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [619765.344687] Lustre: Skipped 11 previous similar messages [620664.360509] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [620664.370868] Lustre: Skipped 17 previous similar messages [621265.373173] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [621265.383527] Lustre: Skipped 11 previous similar messages [622165.396512] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [622165.406855] Lustre: Skipped 17 previous similar messages [623064.420730] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [623064.431076] Lustre: Skipped 17 previous similar messages [623665.438147] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [623665.448498] Lustre: Skipped 11 previous similar messages [624291.473319] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [624291.483584] Lustre: Skipped 17 previous similar messages [643475.002563] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [643475.012825] Lustre: Skipped 12 previous similar messages [643550.138837] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [643550.149096] Lustre: Skipped 1 previous similar message [645195.624238] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [645195.634507] Lustre: Skipped 1 previous similar message [645576.485443] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [645822.232673] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [646293.461086] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [646765.922072] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [647973.305040] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [648039.311465] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [648363.408863] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [648363.419122] Lustre: Skipped 1 previous similar message [651198.202288] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656046.997744] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656051.646624] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656058.094137] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656074.795115] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656074.805377] Lustre: Skipped 1 previous similar message [656098.418094] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656098.428382] Lustre: Skipped 1 previous similar message [656142.505228] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656142.515495] Lustre: Skipped 1 previous similar message [656218.065577] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656218.075845] Lustre: Skipped 10 previous similar messages [656371.469946] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656371.480219] Lustre: Skipped 10 previous similar messages [656721.914974] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656721.925238] Lustre: Skipped 9 previous similar messages [657348.300667] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [657348.310943] Lustre: Skipped 11 previous similar messages [657971.535242] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [657971.545505] Lustre: Skipped 9 previous similar messages [658616.146808] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [658616.157075] Lustre: Skipped 7 previous similar messages [659281.419395] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [659281.429660] Lustre: Skipped 9 previous similar messages [659908.734782] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [659908.745045] Lustre: Skipped 9 previous similar messages [663286.639269] Lustre: fir-OST0005: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [663286.639270] Lustre: fir-OST0007: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [663286.639271] Lustre: fir-OST0001: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [663286.639274] Lustre: Skipped 2 previous similar messages [663286.639274] Lustre: Skipped 2 previous similar messages [663286.680672] Lustre: Skipped 2 previous similar messages [664073.028951] Lustre: fir-OST000b: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [664074.257467] Lustre: fir-OST0003: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [664557.220864] Lustre: fir-OST0003: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [664596.480972] LNetError: 67949:0:(peer.c:2480:lnet_peer_merge_data()) Error deleting NID 10.10.6.33@tcp from peer 10.10.6.33@tcp: -16 [664597.498074] Lustre: fir-OST0001: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [664597.508340] Lustre: Skipped 5 previous similar messages [665148.070525] Lustre: fir-MDT0000-lwp-OST0009: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [665148.086629] Lustre: Skipped 10 previous similar messages [665176.008093] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665176.018295] Lustre: 67985:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548442915/real 1548442918] req@ffff90becd43b900 x1623580834576448/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1548442922 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [665176.046794] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [665177.008113] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [665202.008753] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 18 seconds [665204.008800] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 301 seconds [665204.019141] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [665232.009516] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [665232.019686] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 4 previous similar messages [665252.010028] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 7 seconds [665281.010774] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665281.020947] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 11 previous similar messages [665331.012051] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665331.022226] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [665402.013858] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 21 seconds [665402.024119] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 13 previous similar messages [665532.017196] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665532.027367] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 36 previous similar messages [665572.012800] Lustre: fir-OST000b: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [665602.482133] Lustre: Evicted from MGS (at 10.0.10.51@o2ib7) after server handle changed from 0xffdc3423dd163fef to 0x824cf209dbc07d7d [665602.494267] LustreError: 167-0: fir-MDT0002-lwp-OST0005: This client was evicted by fir-MDT0002; in progress operations using this service will fail. [665602.494291] Lustre: MGC10.0.10.51@o2ib7: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [665602.494293] Lustre: Skipped 11 previous similar messages [665623.140816] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:2665 to 0xb40000402:2689 [665623.140820] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:2661 to 0x800000400:2689 [665623.140821] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:2663 to 0xbc0000402:2689 [665623.140822] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:2665 to 0x540000400:2689 [665623.140824] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:2663 to 0xb00000402:2689 [665623.140894] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:2667 to 0x380000400:2689 [665623.147121] Lustre: fir-OST0001: deleting orphan objects from 0x0:6 to 0x0:33 [665623.147122] Lustre: fir-OST0003: deleting orphan objects from 0x0:5 to 0x0:33 [665623.147141] Lustre: fir-OST0009: deleting orphan objects from 0x0:5 to 0x0:33 [665623.147144] Lustre: fir-OST0005: deleting orphan objects from 0x0:6 to 0x0:33 [665623.147153] Lustre: fir-OST0007: deleting orphan objects from 0x0:6 to 0x0:33 [665623.147242] Lustre: fir-OST000b: deleting orphan objects from 0x0:5 to 0x0:33 [665623.423457] Lustre: fir-OST0001: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [665623.433725] Lustre: Skipped 12 previous similar messages [665652.222455] Lustre: fir-OST000b: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [665673.735315] Lustre: fir-OST0005: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [665673.745575] Lustre: Skipped 1 previous similar message [665737.875886] Lustre: fir-OST0007: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [665923.786655] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665923.797218] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443666/real 1548443666] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443677 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [665924.287217] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665924.297734] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 55256 previous similar messages [665924.308604] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443666/real 1548443666] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443677 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [665924.336162] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 55256 previous similar messages [665925.287248] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665925.297766] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 113115 previous similar messages [665925.309247] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443667/real 1548443667] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443678 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [665925.336767] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 113176 previous similar messages [665927.287294] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665927.297810] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 233498 previous similar messages [665927.309294] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443669/real 1548443669] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443680 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [665927.336819] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 233499 previous similar messages [665931.287399] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665931.297915] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 471698 previous similar messages [665931.309403] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443673/real 1548443673] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443684 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [665931.336905] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 471698 previous similar messages [665939.287600] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665939.298116] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 953542 previous similar messages [665939.309603] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443681/real 1548443681] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443692 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [665939.337125] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 953542 previous similar messages [665955.288017] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665955.298536] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 1913335 previous similar messages [665955.310015] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443697/real 1548443697] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443708 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [665955.337534] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1913324 previous similar messages [665987.288829] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [665987.299350] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 3835743 previous similar messages [665987.310834] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443729/real 1548443729] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443740 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [665987.338357] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3835743 previous similar messages [666051.290470] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666051.300988] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 7692090 previous similar messages [666051.312473] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443793/real 1548443793] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443804 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666051.340000] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7692090 previous similar messages [666124.223345] LNet: Service thread pid 68301 was inactive for 200.43s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [666124.240363] Pid: 68301, comm: ll_ost00_001 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [666124.250444] Call Trace: [666124.252991] [] 0xffffffffffffffff [666124.258102] LustreError: dumping log to /tmp/lustre-log.1548443866.68301 [666179.293744] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666179.304261] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 15351166 previous similar messages [666179.315748] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548443921/real 1548443921] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548443932 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666179.343269] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 15351156 previous similar messages [666416.410177] Lustre: fir-OST0003: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [666416.420439] Lustre: Skipped 1 previous similar message [666435.300300] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666435.310824] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 30876806 previous similar messages [666435.322303] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444177/real 1548444177] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548444188 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666435.349840] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 30876805 previous similar messages [666518.557444] Lustre: 68302:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff90abf6038850 x1623580998927024/t0(0) o101->c680654c-e7b5-e476-0d23-f02ec53e73ba@10.0.10.3@o2ib7:421/0 lens 328/400 e 24 to 0 dl 1548444266 ref 2 fl Interpret:/0/0 rc 0/0 [666526.177271] Lustre: fir-OST0001: Client c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) reconnecting [666526.187410] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [666526.197755] Lustre: Skipped 2 previous similar messages [666796.548575] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [666947.313400] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666947.323919] LNetError: 68301:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 61648595 previous similar messages [666947.335411] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444689/real 1548444689] req@ffff90a30f4fe900 x1623580834592048/t0(0) o106->fir-OST0001@10.10.6.33@tcp:15/16 lens 296/280 e 0 to 1 dl 1548444700 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666947.362932] Lustre: 68301:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 61648597 previous similar messages [667127.405777] Lustre: fir-OST0001: Client c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) reconnecting [667127.415918] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [667179.965465] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [667179.975745] Lustre: Skipped 1 previous similar message [667288.108966] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [667435.065900] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 6 seconds [667435.076072] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [667435.085593] Lustre: fir-MDT0001-lwp-OST0007: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [667435.101665] Lustre: Skipped 7 previous similar messages [667436.065994] Lustre: fir-MDT0001-lwp-OST0003: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [667436.082062] Lustre: Skipped 3 previous similar messages [667438.066013] Lustre: fir-MDT0001-lwp-OST000b: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [667489.067282] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 15 seconds [667489.077541] LNet: 67915:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 23 previous similar messages [667510.228292] Lustre: Failing over fir-OST0007 [667510.228293] Lustre: Failing over fir-OST000b [667510.228391] LNet: Service thread pid 68301 completed after 1586.40s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [667510.253363] Lustre: Skipped 4 previous similar messages [667511.093233] Lustre: server umount fir-OST0003 complete [667511.098464] Lustre: Skipped 3 previous similar messages [667516.811663] md11: detected capacity change from 64011422924800 to 0 [667516.815873] md1: detected capacity change from 64011422924800 to 0 [667516.815883] md: md1 stopped. [667516.827287] md: md11 stopped. [667516.837728] md9: detected capacity change from 64011422924800 to 0 [667516.844023] md: md9 stopped. [667516.984461] md7: detected capacity change from 64011422924800 to 0 [667516.990744] md: md7 stopped. [667517.674300] md3: detected capacity change from 64011422924800 to 0 [667517.680584] md: md3 stopped. [667517.802371] md5: detected capacity change from 64011422924800 to 0 [667517.808651] md: md5 stopped. [667521.157101] LNet: Removed LNI 10.0.10.102@o2ib7 [667631.333452] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [667631.341218] alg: No test for adler32 (adler32-zlib) [667632.143326] Lustre: Lustre: Build Version: 2.12.0 [667632.268948] LNet: Using FastReg for registration [667632.285687] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [667642.217174] md: md7 stopped. [667642.266643] md/raid:md7: device dm-92 operational as raid disk 0 [667642.272741] md/raid:md7: device dm-38 operational as raid disk 9 [667642.278837] md/raid:md7: device dm-49 operational as raid disk 8 [667642.284936] md/raid:md7: device dm-97 operational as raid disk 7 [667642.291029] md/raid:md7: device dm-111 operational as raid disk 6 [667642.297216] md/raid:md7: device dm-96 operational as raid disk 5 [667642.303312] md/raid:md7: device dm-101 operational as raid disk 4 [667642.309498] md/raid:md7: device dm-91 operational as raid disk 3 [667642.315611] md/raid:md7: device dm-86 operational as raid disk 2 [667642.321710] md/raid:md7: device dm-109 operational as raid disk 1 [667642.328683] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [667642.347765] md7: detected capacity change from 0 to 64011422924800 [667642.363411] md: md1 stopped. [667642.387838] md/raid:md1: device dm-26 operational as raid disk 0 [667642.393954] md/raid:md1: device dm-9 operational as raid disk 9 [667642.399967] md/raid:md1: device dm-16 operational as raid disk 8 [667642.406065] md/raid:md1: device dm-59 operational as raid disk 7 [667642.412158] md/raid:md1: device dm-23 operational as raid disk 6 [667642.418252] md/raid:md1: device dm-60 operational as raid disk 5 [667642.424353] md/raid:md1: device dm-34 operational as raid disk 4 [667642.430454] md/raid:md1: device dm-68 operational as raid disk 3 [667642.436555] md/raid:md1: device dm-43 operational as raid disk 2 [667642.442648] md/raid:md1: device dm-57 operational as raid disk 1 [667642.449423] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [667642.468197] md1: detected capacity change from 0 to 64011422924800 [667642.482995] md: md3 stopped. [667642.507284] md/raid:md3: device dm-2 operational as raid disk 0 [667642.513306] md/raid:md3: device dm-0 operational as raid disk 9 [667642.519322] md/raid:md3: device dm-65 operational as raid disk 8 [667642.525423] md/raid:md3: device dm-24 operational as raid disk 7 [667642.531527] md/raid:md3: device dm-14 operational as raid disk 6 [667642.537634] md/raid:md3: device dm-118 operational as raid disk 5 [667642.543825] md/raid:md3: device dm-64 operational as raid disk 4 [667642.549937] md/raid:md3: device dm-75 operational as raid disk 3 [667642.556041] md/raid:md3: device dm-31 operational as raid disk 2 [667642.562150] md/raid:md3: device dm-15 operational as raid disk 1 [667642.569488] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [667642.588621] md3: detected capacity change from 0 to 64011422924800 [667642.606293] md: md9 stopped. [667642.682594] md/raid:md9: device dm-47 operational as raid disk 0 [667642.688712] md/raid:md9: device dm-42 operational as raid disk 9 [667642.694819] md/raid:md9: device dm-39 operational as raid disk 8 [667642.700918] md/raid:md9: device dm-28 operational as raid disk 7 [667642.707023] md/raid:md9: device dm-116 operational as raid disk 6 [667642.713216] md/raid:md9: device dm-71 operational as raid disk 5 [667642.719320] md/raid:md9: device dm-110 operational as raid disk 4 [667642.725507] md/raid:md9: device dm-94 operational as raid disk 3 [667642.731607] md/raid:md9: device dm-33 operational as raid disk 2 [667642.737707] md/raid:md9: device dm-53 operational as raid disk 1 [667642.744624] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [667642.785126] md9: detected capacity change from 0 to 64011422924800 [667642.807357] md: md11 stopped. [667642.894906] md/raid:md11: device dm-50 operational as raid disk 0 [667642.901101] md/raid:md11: device dm-46 operational as raid disk 9 [667642.907298] md/raid:md11: device dm-30 operational as raid disk 8 [667642.913498] md/raid:md11: device dm-112 operational as raid disk 7 [667642.919785] md/raid:md11: device dm-114 operational as raid disk 6 [667642.926063] md/raid:md11: device dm-80 operational as raid disk 5 [667642.932257] md/raid:md11: device dm-48 operational as raid disk 4 [667642.938456] md/raid:md11: device dm-40 operational as raid disk 3 [667642.944652] md/raid:md11: device dm-104 operational as raid disk 2 [667642.950935] md/raid:md11: device dm-95 operational as raid disk 1 [667642.958127] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [667643.018257] md11: detected capacity change from 0 to 64011422924800 [667643.036014] md: md5 stopped. [667643.064543] md/raid:md5: device dm-84 operational as raid disk 0 [667643.070673] md/raid:md5: device dm-8 operational as raid disk 9 [667643.076692] md/raid:md5: device dm-3 operational as raid disk 8 [667643.082716] md/raid:md5: device dm-22 operational as raid disk 7 [667643.088828] md/raid:md5: device dm-21 operational as raid disk 6 [667643.094934] md/raid:md5: device dm-67 operational as raid disk 5 [667643.101030] md/raid:md5: device dm-36 operational as raid disk 4 [667643.107132] md/raid:md5: device dm-10 operational as raid disk 3 [667643.113238] md/raid:md5: device dm-76 operational as raid disk 2 [667643.119334] md/raid:md5: device dm-56 operational as raid disk 1 [667643.126178] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [667643.146418] md5: detected capacity change from 0 to 64011422924800 [667643.255429] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [667643.509481] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [667643.581391] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [667643.626727] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667643.791451] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [667643.850379] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667643.940999] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667644.135584] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667644.496437] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [667644.551062] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [667644.817397] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667644.865127] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667645.268306] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 7 seconds [667647.268333] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 9 seconds [667650.268407] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 12 seconds [667650.278666] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [667651.271621] Lustre: fir-OST0007: Imperative Recovery not enabled, recovery window 300-900 [667651.280578] Lustre: fir-OST0007: in recovery but waiting for the first client to connect [667653.268486] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 15 seconds [667653.278740] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [667656.271323] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [667656.280262] Lustre: fir-OST0003: in recovery but waiting for the first client to connect [667657.418798] Lustre: fir-OST0007: Will be in recovery for at least 5:00, or until 6 clients reconnect [667657.428589] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [667658.268618] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [667658.278880] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [667661.182210] Lustre: fir-OST0003: Will be in recovery for at least 5:00, or until 5 clients reconnect [667661.191471] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [667661.201831] Lustre: Skipped 1 previous similar message [667661.271365] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [667661.280212] Lustre: fir-OST0001: in recovery but waiting for the first client to connect [667662.982465] Lustre: fir-OST0001: Will be in recovery for at least 5:00, or until 6 clients reconnect [667662.982496] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [667662.982681] Lustre: fir-OST0007: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [667662.982683] Lustre: fir-OST0003: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [667666.271468] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [667666.280313] Lustre: fir-OST0009: in recovery but waiting for the first client to connect [667667.268844] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 29 seconds [667667.279098] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [667667.582333] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [667667.584855] Lustre: fir-OST0009: Will be in recovery for at least 5:00, or until 6 clients reconnect [667667.601908] Lustre: Skipped 4 previous similar messages [667669.812839] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [667669.830322] LustreError: Skipped 1 previous similar message [667671.271037] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [667671.279999] Lustre: fir-OST000b: in recovery but waiting for the first client to connect [667673.634697] Lustre: fir-OST000b: Will be in recovery for at least 5:00, or until 5 clients reconnect [667675.796504] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [667675.806884] Lustre: Skipped 8 previous similar messages [667678.548599] Lustre: fir-OST0003: Recovery over after 0:17, of 5 clients 5 recovered and 0 were evicted. [667678.717076] Lustre: fir-OST0009: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 6 known clients (4 recovered, 1 in progress, and 0 evicted) already passed deadline 0:10 [667678.736113] Lustre: Skipped 1 previous similar message [667679.687239] Lustre: fir-OST0009: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 6 known clients (4 recovered, 1 in progress, and 0 evicted) already passed deadline 0:11 [667679.706306] Lustre: Skipped 16 previous similar messages [667703.973001] Lustre: fir-OST0009: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 6 known clients (4 recovered, 1 in progress, and 0 evicted) already passed deadline 0:36 [667703.992038] Lustre: Skipped 20 previous similar messages [667729.061745] Lustre: fir-OST0009: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 6 known clients (4 recovered, 1 in progress, and 0 evicted) already passed deadline 1:01 [667729.080765] Lustre: Skipped 15 previous similar messages [667729.872765] Lustre: fir-OST000b: deleting orphan objects from 0x0:68 to 0x0:97 [667729.899482] Lustre: fir-OST0003: deleting orphan objects from 0x0:68 to 0x0:97 [667729.928962] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [667729.939244] Lustre: Skipped 19 previous similar messages [667729.972588] Lustre: fir-OST0009: Recovery over after 1:02, of 6 clients 6 recovered and 0 were evicted. [667729.982084] Lustre: Skipped 1 previous similar message [667730.084185] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:2731 to 0x380000400:2753 [667730.119039] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:2732 to 0xbc0000402:2753 [667730.916058] Lustre: fir-OST000b: deleting orphan objects from 0x380000402:3 to 0x380000402:33 [667730.938994] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000401:3 to 0xbc0000401:33 [667731.017681] Lustre: fir-OST0007: Recovery over after 1:14, of 6 clients 6 recovered and 0 were evicted. [667731.027197] Lustre: Skipped 1 previous similar message [667754.150892] Lustre: fir-OST0009: deleting orphan objects from 0x0:67 to 0x0:97 [667754.151042] Lustre: fir-OST0007: deleting orphan objects from 0x0:67 to 0x0:97 [667754.154087] Lustre: fir-OST0005: deleting orphan objects from 0x0:67 to 0x0:97 [667754.154663] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:2732 to 0x540000400:2753 [667754.160138] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:2732 to 0xb00000402:2753 [667754.172391] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:2732 to 0xb40000402:2753 [667754.176798] Lustre: fir-OST0001: deleting orphan objects from 0x0:68 to 0x0:97 [667754.267494] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:2733 to 0x800000400:2753 [667820.325873] Lustre: fir-OST0001: Connection restored to (at 10.8.9.6@o2ib6) [667820.333016] Lustre: Skipped 21 previous similar messages [668093.730131] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [668093.737280] Lustre: Skipped 1 previous similar message [668425.746297] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [668425.756555] Lustre: Skipped 14 previous similar messages [668772.028979] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [668772.039249] Lustre: Skipped 5 previous similar messages [669483.632682] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [669483.642950] Lustre: Skipped 3 previous similar messages [670135.307750] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [670135.318020] Lustre: Skipped 21 previous similar messages [670812.804326] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [670812.814681] Lustre: Skipped 2 previous similar messages [671455.855839] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [671455.866097] Lustre: Skipped 13 previous similar messages [672675.447849] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [672675.458146] Lustre: Skipped 2 previous similar messages [672880.377986] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [672880.388266] Lustre: Skipped 1 previous similar message [673196.077717] Lustre: fir-OST0009: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [673196.088090] Lustre: Skipped 1 previous similar message [673551.553885] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [673551.564157] Lustre: Skipped 5 previous similar messages [674189.394239] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [674189.404497] Lustre: Skipped 11 previous similar messages [674803.555786] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [674803.566056] Lustre: Skipped 9 previous similar messages [675414.664567] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [675414.674832] Lustre: Skipped 12 previous similar messages [676048.062700] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [676048.073044] Lustre: Skipped 20 previous similar messages [676665.117442] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [676665.127792] Lustre: Skipped 18 previous similar messages [677275.137797] Lustre: fir-OST0001: Connection restored to (at 10.8.9.6@o2ib6) [677275.144942] Lustre: Skipped 15 previous similar messages [677905.823558] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [677905.833927] Lustre: Skipped 20 previous similar messages [678530.716448] Lustre: fir-OST0001: Connection restored to (at 10.8.9.6@o2ib6) [678530.723588] Lustre: Skipped 20 previous similar messages [679146.818086] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [679146.828443] Lustre: Skipped 26 previous similar messages [679776.005753] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [679776.016099] Lustre: Skipped 27 previous similar messages [680384.894801] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [680384.901944] Lustre: Skipped 31 previous similar messages [680985.668936] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [680985.679286] Lustre: Skipped 43 previous similar messages [681587.003517] Lustre: fir-OST0005: Connection restored to (at 10.8.9.6@o2ib6) [681587.010672] Lustre: Skipped 19 previous similar messages [682187.552386] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [682187.559534] Lustre: Skipped 46 previous similar messages [682868.338653] Lustre: fir-OST000b: Connection restored to (at 10.8.9.6@o2ib6) [682868.345793] Lustre: Skipped 40 previous similar messages [683502.314198] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [683502.321350] Lustre: Skipped 16 previous similar messages [685731.752691] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [685731.762961] Lustre: Skipped 2 previous similar messages [686776.256238] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [686948.145515] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [687093.964047] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [687206.666685] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [687295.394576] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [687538.914416] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [687960.091965] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [687960.102232] Lustre: Skipped 2 previous similar messages [688781.304582] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [688781.314845] Lustre: Skipped 1 previous similar message [689408.802637] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [689408.812905] Lustre: Skipped 3 previous similar messages [690043.916389] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [690043.926656] Lustre: Skipped 10 previous similar messages [690661.402787] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [690661.413050] Lustre: Skipped 8 previous similar messages [691296.570869] Lustre: fir-OST0005: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [691296.581217] Lustre: Skipped 17 previous similar messages [691906.645917] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [691906.656182] Lustre: Skipped 18 previous similar messages [692559.242700] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [692559.253058] Lustre: Skipped 12 previous similar messages [693177.169864] Lustre: fir-OST0005: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [693177.180209] Lustre: Skipped 17 previous similar messages [693852.604850] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [693852.615114] Lustre: Skipped 22 previous similar messages [694463.764789] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [694463.775140] Lustre: Skipped 13 previous similar messages [695105.886408] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [695105.896684] Lustre: Skipped 19 previous similar messages [695735.780793] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [695735.791145] Lustre: Skipped 18 previous similar messages [696338.776532] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [696338.786895] Lustre: Skipped 14 previous similar messages [696965.963978] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [696965.974240] Lustre: Skipped 10 previous similar messages [697572.357738] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [697572.368088] Lustre: Skipped 14 previous similar messages [698247.214419] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [698247.224703] Lustre: Skipped 15 previous similar messages [698867.214214] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [698867.224501] Lustre: Skipped 12 previous similar messages [699557.858738] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [699557.868999] Lustre: Skipped 6 previous similar messages [700224.705446] Lustre: fir-OST0005: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [700224.715796] Lustre: Skipped 11 previous similar messages [700833.099620] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [700833.109941] Lustre: Skipped 14 previous similar messages [701452.534926] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [701452.545186] Lustre: Skipped 5 previous similar messages [702077.381823] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [702077.392086] Lustre: Skipped 5 previous similar messages [702685.556601] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [702685.566869] Lustre: Skipped 12 previous similar messages [702726.476130] Lustre: 79951:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548480466/real 1548480467] req@ffff90bf46656900 x1623662662890224/t0(0) o400->fir-MDT0003-lwp-OST0009@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548480473 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [702726.476216] Lustre: fir-MDT0001-lwp-OST0009: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [702726.476218] Lustre: Skipped 2 previous similar messages [702726.526392] Lustre: 79951:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [702732.096331] Lustre: 79953:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548480466/real 1548480468] req@ffff90bf46650c00 x1623662662890272/t0(0) o400->fir-MDT0001-lwp-OST000b@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548480473 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [702732.097342] Lustre: fir-MDT0003-lwp-OST0005: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [702732.097344] Lustre: Skipped 6 previous similar messages [702732.146169] Lustre: 79953:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [702732.166334] LNetError: 79895:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [702732.176419] LNetError: 79895:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.52@o2ib7 (6): c: 0, oc: 0, rc: 8 [702753.359532] Lustre: fir-OST0003: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [702753.359533] Lustre: fir-OST0001: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [702753.359536] Lustre: Skipped 3 previous similar messages [702753.382968] Lustre: Skipped 7 previous similar messages [702776.630878] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [702776.648248] LustreError: Skipped 16 previous similar messages [702801.719505] Lustre: fir-OST000b: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [702801.720249] Lustre: fir-OST0009: deleting orphan objects from 0x540000401:14172 to 0x540000401:14209 [702801.720268] Lustre: fir-OST0005: deleting orphan objects from 0x800000401:14178 to 0x800000401:14209 [702801.720364] Lustre: fir-OST0007: deleting orphan objects from 0xb00000400:14178 to 0xb00000400:14209 [702801.720568] Lustre: fir-OST0001: deleting orphan objects from 0xb40000400:14191 to 0xb40000400:14209 [702801.720660] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000400:14179 to 0xbc0000400:14209 [702801.721199] Lustre: fir-OST000b: deleting orphan objects from 0x380000402:26088 to 0x380000402:26113 [702801.721975] Lustre: fir-OST0009: deleting orphan objects from 0x540000402:26100 to 0x540000402:26145 [702801.721985] Lustre: fir-OST0005: deleting orphan objects from 0x800000402:26015 to 0x800000402:26049 [702801.722134] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000401:26088 to 0xbc0000401:26113 [702801.722195] Lustre: fir-OST0001: deleting orphan objects from 0xb40000401:26006 to 0xb40000401:26049 [702801.722287] Lustre: fir-OST0007: deleting orphan objects from 0xb00000401:26016 to 0xb00000401:26049 [702801.829975] Lustre: Skipped 11 previous similar messages [702801.835538] Lustre: fir-OST000b: deleting orphan objects from 0x380000401:14164 to 0x380000401:14209 [703308.166953] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [703308.177298] Lustre: Skipped 75 previous similar messages [703959.970011] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [703959.980364] Lustre: Skipped 9 previous similar messages [704685.873143] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [704685.883399] Lustre: Skipped 13 previous similar messages [706016.551581] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [706016.561935] Lustre: Skipped 2 previous similar messages [706126.918525] Lustre: fir-OST0005: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [706126.928976] Lustre: Skipped 2 previous similar messages [706539.140535] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [706539.150802] Lustre: Skipped 6 previous similar messages [706898.082238] Lustre: fir-OST0005: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [707803.776241] Lustre: fir-OST0009: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [707803.786588] Lustre: Skipped 5 previous similar messages [711055.292797] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [711243.192186] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [711831.229747] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [711831.240038] Lustre: Skipped 2 previous similar messages [712184.019108] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [712184.029367] Lustre: Skipped 2 previous similar messages [713343.478732] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [713343.488999] Lustre: Skipped 1 previous similar message [714069.382245] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [714069.392514] Lustre: Skipped 6 previous similar messages [714968.406422] Lustre: fir-OST000b: Client f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) reconnecting [714968.416541] Lustre: fir-OST000b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [714968.426902] Lustre: Skipped 5 previous similar messages [715089.092106] Lustre: fir-OST0005: Client 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) reconnecting [715089.102283] Lustre: Skipped 2 previous similar messages [715107.354546] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.3.11@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [715107.371834] LustreError: Skipped 2 previous similar messages [715132.379379] Lustre: fir-OST0005: Client 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) reconnecting [715132.389471] Lustre: Skipped 2 previous similar messages [715710.308129] Lustre: fir-OST000b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [715710.318540] Lustre: Skipped 10 previous similar messages [718054.859068] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [718413.641999] Lustre: fir-OST0003: Client f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) reconnecting [718413.652123] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [718413.947712] LustreError: 84256:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff90abf7f5a850 x1623671563143856/t0(0) o4->f9afcafe-b7fa-076a-0d87-fc0f78fd02d0@10.8.6.20@o2ib6:245/0 lens 488/448 e 1 to 0 dl 1548496185 ref 1 fl Interpret:/0/0 rc 0/0 [718413.972357] Lustre: fir-OST0003: Bulk IO write error with f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6), client will retry: rc = -110 [718420.206671] Lustre: fir-OST0005: Client 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) reconnecting [718421.604909] LustreError: 84256:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff90abf7f5c050 x1623671563143856/t0(0) o4->f9afcafe-b7fa-076a-0d87-fc0f78fd02d0@10.8.6.20@o2ib6:246/0 lens 488/448 e 1 to 0 dl 1548496186 ref 1 fl Interpret:/2/0 rc 0/0 [718421.629719] Lustre: fir-OST0003: Bulk IO write error with f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6), client will retry: rc = -110 [718445.569822] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [718445.587190] LustreError: Skipped 6 previous similar messages [720796.659803] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [720796.670076] Lustre: Skipped 15 previous similar messages [721039.803699] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721240.056042] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721467.198594] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721467.208853] Lustre: Skipped 2 previous similar messages [721633.261509] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [722106.645996] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [722106.656257] Lustre: Skipped 4 previous similar messages [722824.272911] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [722824.283177] Lustre: Skipped 7 previous similar messages [726813.891292] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [726813.901551] Lustre: Skipped 7 previous similar messages [726903.476383] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [727109.239580] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [727109.249849] Lustre: Skipped 1 previous similar message [740167.196331] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [740167.206593] Lustre: Skipped 1 previous similar message [744391.287884] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [744391.298155] Lustre: Skipped 1 previous similar message [745211.588179] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [745211.598470] Lustre: Skipped 1 previous similar message [745855.671935] Lustre: fir-OST0005: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [745894.958620] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [747013.953151] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747087.502988] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747087.513265] Lustre: Skipped 5 previous similar messages [747166.246372] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747166.256641] Lustre: Skipped 1 previous similar message [747221.972018] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747221.982281] Lustre: Skipped 1 previous similar message [747425.369622] Lustre: fir-OST000b: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [747425.379968] Lustre: Skipped 1 previous similar message [747433.915158] Lustre: fir-OST0001: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [747433.925510] Lustre: Skipped 1 previous similar message [747560.549862] Lustre: fir-OST0007: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [748449.226261] Lustre: DEBUG MARKER: Sat Jan 26 10:09:49 2019 [748665.829887] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [748670.596728] Lustre: fir-OST0009: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [748670.607085] Lustre: Skipped 2 previous similar messages [748766.674235] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [748766.681381] Lustre: Skipped 2 previous similar messages [748787.234517] Lustre: fir-OST0001: Connection restored to (at 10.8.9.6@o2ib6) [748787.241656] Lustre: Skipped 3 previous similar messages [748836.628261] Lustre: fir-OST0007: Connection restored to (at 10.8.9.6@o2ib6) [748992.853470] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [751469.789894] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [751469.800434] Lustre: Skipped 5 previous similar messages [752236.742725] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [752243.456945] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [752539.494655] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [752539.505001] Lustre: Skipped 5 previous similar messages [752598.616553] Lustre: fir-OST0003: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [752942.662742] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [752942.673097] Lustre: Skipped 4 previous similar messages [753630.198638] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [753807.860626] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [753932.954580] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [754067.582762] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [754170.430919] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [754254.085480] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [754429.026194] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [754739.048748] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [754739.059014] Lustre: Skipped 2 previous similar messages [755398.410254] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [755398.420606] Lustre: Skipped 6 previous similar messages [756405.230148] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [756405.240493] Lustre: Skipped 8 previous similar messages [757717.529594] Lustre: fir-OST0005: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [757717.531052] Lustre: fir-OST0001: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [757717.531055] Lustre: Skipped 5 previous similar messages [757717.553253] Lustre: Skipped 4 previous similar messages [757717.568478] Lustre: fir-OST0001: deleting orphan objects from 0x0:47243 to 0x0:47265 [757717.570173] Lustre: fir-OST0007: deleting orphan objects from 0x0:47249 to 0x0:47265 [757717.580850] Lustre: fir-OST0009: deleting orphan objects from 0x0:47229 to 0x0:47265 [757717.582272] Lustre: fir-OST0003: deleting orphan objects from 0x0:47234 to 0x0:47265 [757717.584521] Lustre: fir-OST000b: deleting orphan objects from 0x0:47228 to 0x0:47265 [757717.587240] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:72787 to 0x380000400:72833 [757717.587609] Lustre: fir-OST0005: deleting orphan objects from 0x0:47238 to 0x0:47265 [757717.588142] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:72788 to 0x540000400:72833 [757944.784832] Lustre: fir-OST0005: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [757944.795175] Lustre: Skipped 7 previous similar messages [758197.670993] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [758197.681255] Lustre: Skipped 3 previous similar messages [758609.704264] Lustre: fir-OST0007: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [758609.714612] Lustre: Skipped 5 previous similar messages [759220.906831] Lustre: fir-OST0009: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [759220.917182] Lustre: Skipped 11 previous similar messages [759843.948987] Lustre: fir-OST0009: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [759843.959340] Lustre: Skipped 26 previous similar messages [760654.396925] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [760654.407200] Lustre: Skipped 17 previous similar messages [761455.462321] Lustre: fir-OST0009: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [761455.472669] Lustre: Skipped 2 previous similar messages [762536.342157] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [763593.699814] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [764245.452601] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [764245.463130] Lustre: Skipped 8 previous similar messages [765945.923466] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [766877.534721] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [767479.085106] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [767827.829197] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [768086.593957] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [768231.087171] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [769135.946812] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [769155.014879] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [769196.423142] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [769196.433666] Lustre: Skipped 3 previous similar messages [769654.654021] Lustre: fir-OST0007: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [771369.961865] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [771420.375102] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [771775.604946] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [772351.803666] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [772747.633222] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [773598.654389] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [774265.280187] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [775055.804731] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [775055.814995] Lustre: Skipped 1 previous similar message [776465.177292] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [776465.187814] Lustre: Skipped 1 previous similar message [776642.827202] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [777312.838535] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [778209.133865] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [778209.144174] Lustre: Skipped 2 previous similar messages [779196.907404] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [779196.917667] Lustre: Skipped 1 previous similar message [779846.446174] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [779846.456444] Lustre: Skipped 2 previous similar messages [780602.980863] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [780602.991133] Lustre: Skipped 5 previous similar messages [781236.114185] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [781236.124458] Lustre: Skipped 1 previous similar message [781862.106571] Lustre: fir-OST0009: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [781862.116921] Lustre: Skipped 1 previous similar message [782579.084466] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [782579.094731] Lustre: Skipped 5 previous similar messages [783365.244279] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [783365.254560] Lustre: Skipped 6 previous similar messages [784317.100682] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [784317.110952] Lustre: Skipped 3 previous similar messages [786245.272095] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [786401.266052] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [788855.548630] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [788855.558896] Lustre: Skipped 2 previous similar messages [788952.999331] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [788953.009596] Lustre: Skipped 1 previous similar message [789361.072289] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [789478.662127] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [789631.381139] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [789631.391628] Lustre: Skipped 1 previous similar message [790659.543964] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [790659.554483] Lustre: Skipped 1 previous similar message [790763.535666] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [790860.644603] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [793950.337228] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [793976.622895] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [793976.633470] Lustre: Skipped 1 previous similar message [794070.454434] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [794147.053295] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [794147.063822] Lustre: Skipped 2 previous similar messages [794353.945378] Lustre: fir-OST0005: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [794353.955729] Lustre: Skipped 4 previous similar messages [794986.565522] Lustre: fir-OST0003: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [794986.575869] Lustre: Skipped 2 previous similar messages [795823.976701] Lustre: fir-OST0007: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [795823.987079] Lustre: Skipped 4 previous similar messages [797020.647444] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [797020.657708] Lustre: Skipped 4 previous similar messages [801863.120255] md: data-check of RAID array md5 [801869.086094] md: data-check of RAID array md11 [801875.162668] md: data-check of RAID array md9 [801881.260363] md: data-check of RAID array md3 [801887.329901] md: data-check of RAID array md1 [801893.409513] md: data-check of RAID array md7 [802028.909833] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802185.988116] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802185.998638] Lustre: Skipped 1 previous similar message [802336.310929] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802655.648546] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802655.659069] Lustre: Skipped 4 previous similar messages [805901.816878] Lustre: fir-OST0009: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [805901.827145] Lustre: Skipped 5 previous similar messages [809413.422964] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [809413.433491] Lustre: Skipped 5 previous similar messages [809471.695068] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [809471.705596] Lustre: Skipped 1 previous similar message [809841.710435] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [810750.231115] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [810750.241638] Lustre: Skipped 1 previous similar message [810938.104140] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [811865.645956] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [814136.038497] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814214.799828] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814373.402917] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814425.062137] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814474.240355] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814493.007725] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [814675.324351] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814932.989774] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814933.000299] Lustre: Skipped 1 previous similar message [815190.266451] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815190.276971] Lustre: Skipped 1 previous similar message [815336.971595] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815336.982133] Lustre: Skipped 3 previous similar messages [815595.339054] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815595.349577] Lustre: Skipped 12 previous similar messages [816243.578211] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [816243.588729] Lustre: Skipped 9 previous similar messages [817251.802675] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [817251.813199] Lustre: Skipped 9 previous similar messages [817888.952857] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [817888.963406] Lustre: Skipped 12 previous similar messages [818583.238368] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [818583.248894] Lustre: Skipped 8 previous similar messages [821589.026736] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [824995.418529] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [824995.428794] Lustre: Skipped 5 previous similar messages [826350.772835] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [826350.783100] Lustre: Skipped 1 previous similar message [826389.274448] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [826585.710471] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [826813.007463] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827340.362116] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827489.088631] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827723.722674] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827881.317355] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [828530.728342] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [828530.738615] Lustre: Skipped 2 previous similar messages [829191.876103] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [829879.632595] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [829879.642876] Lustre: Skipped 3 previous similar messages [830527.858341] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [830527.868601] Lustre: Skipped 1 previous similar message [831161.241413] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [831161.251686] Lustre: Skipped 2 previous similar messages [832514.801789] Lustre: fir-OST0003: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [832514.812139] Lustre: Skipped 7 previous similar messages [834989.395568] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [835076.519486] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [835321.972778] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [835466.483064] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [835466.493328] Lustre: Skipped 1 previous similar message [835618.862875] Lustre: fir-OST0009: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [835778.158347] Lustre: fir-OST0005: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [835778.168703] Lustre: Skipped 2 previous similar messages [836088.167144] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [836088.177412] Lustre: Skipped 3 previous similar messages [836689.953852] Lustre: fir-OST0009: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [836689.964201] Lustre: Skipped 2 previous similar messages [837303.654756] Lustre: fir-OST0009: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [837303.665102] Lustre: Skipped 41 previous similar messages [837950.769692] Lustre: fir-OST000b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [837950.780041] Lustre: Skipped 28 previous similar messages [838591.644491] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [838591.654756] Lustre: Skipped 26 previous similar messages [839241.683377] Lustre: fir-OST0003: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [839241.693729] Lustre: Skipped 32 previous similar messages [839889.767579] Lustre: fir-OST000b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [839889.777925] Lustre: Skipped 23 previous similar messages [840538.823228] Lustre: fir-OST000b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [840538.833486] Lustre: Skipped 26 previous similar messages [841139.756791] Lustre: fir-OST0009: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [841139.767148] Lustre: Skipped 24 previous similar messages [841743.664412] Lustre: fir-OST000b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [841743.674760] Lustre: Skipped 15 previous similar messages [842383.197371] Lustre: fir-OST0009: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [842383.207746] Lustre: Skipped 20 previous similar messages [843248.898867] Lustre: fir-OST0005: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [843248.909216] Lustre: Skipped 15 previous similar messages [843922.040496] Lustre: fir-OST0009: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [843922.050849] Lustre: Skipped 7 previous similar messages [844734.577276] Lustre: fir-OST0007: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [844734.587628] Lustre: Skipped 2 previous similar messages [845364.733149] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [845364.743504] Lustre: Skipped 1 previous similar message [846386.702851] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [847013.600862] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [847013.611118] Lustre: Skipped 31 previous similar messages [847633.855586] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [847633.866132] Lustre: Skipped 37 previous similar messages [848264.963314] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [848264.973831] Lustre: Skipped 34 previous similar messages [848868.240030] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [848868.250558] Lustre: Skipped 25 previous similar messages [849481.441227] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [849481.451748] Lustre: Skipped 23 previous similar messages [850090.626685] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [850090.637272] Lustre: Skipped 44 previous similar messages [850725.429863] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [850725.440383] Lustre: Skipped 6 previous similar messages [851376.710545] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [851376.721073] Lustre: Skipped 13 previous similar messages [851993.991064] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [851994.001594] Lustre: Skipped 25 previous similar messages [852602.409291] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [852602.419806] Lustre: Skipped 22 previous similar messages [853246.634547] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [853246.645075] Lustre: Skipped 19 previous similar messages [853909.646076] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [853909.656604] Lustre: Skipped 16 previous similar messages [854533.145774] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [854533.156290] Lustre: Skipped 17 previous similar messages [855241.045703] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [855241.056224] Lustre: Skipped 24 previous similar messages [856123.004485] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [856123.014766] Lustre: Skipped 2 previous similar messages [856916.837020] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [856916.847321] Lustre: Skipped 1 previous similar message [857670.565036] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [857670.575439] Lustre: Skipped 15 previous similar messages [858270.581323] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [858270.591687] Lustre: Skipped 12 previous similar messages [858870.597018] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [858870.607366] Lustre: Skipped 11 previous similar messages [859470.613183] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [859470.623583] Lustre: Skipped 12 previous similar messages [860071.630196] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [860071.640547] Lustre: Skipped 13 previous similar messages [860971.668469] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [860971.678847] Lustre: Skipped 17 previous similar messages [861870.690990] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [861870.701346] Lustre: Skipped 17 previous similar messages [862470.710213] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [862470.720569] Lustre: Skipped 15 previous similar messages [863140.444615] Lustre: fir-OST0007: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [863140.454965] Lustre: Skipped 2 previous similar messages [864324.465664] Lustre: fir-OST0001: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [864324.475928] Lustre: Skipped 4 previous similar messages [866042.916426] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [866042.926950] Lustre: Skipped 3 previous similar messages [866138.713066] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [866138.723596] Lustre: Skipped 2 previous similar messages [866413.741773] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [866413.752299] Lustre: Skipped 1 previous similar message [866967.138249] Lustre: fir-OST000b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [868207.252106] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [868207.262375] Lustre: Skipped 2 previous similar messages [868640.569416] Lustre: fir-OST0005: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [868886.554145] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [868886.564404] Lustre: Skipped 3 previous similar messages [869264.693332] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [869264.703593] Lustre: Skipped 3 previous similar messages [869877.833607] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [869877.843872] Lustre: Skipped 8 previous similar messages [870509.145358] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [870509.155625] Lustre: Skipped 4 previous similar messages [871293.059198] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [871293.069464] Lustre: Skipped 6 previous similar messages [872022.075798] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [872022.086064] Lustre: Skipped 6 previous similar messages [872684.458606] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [872684.468873] Lustre: Skipped 13 previous similar messages [873347.024275] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [873347.034537] Lustre: Skipped 5 previous similar messages [873964.598470] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [873964.608738] Lustre: Skipped 6 previous similar messages [875091.758581] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [875091.769107] Lustre: Skipped 6 previous similar messages [875864.648282] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [875864.658545] Lustre: Skipped 8 previous similar messages [876466.723836] Lustre: fir-OST0005: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [876466.734192] Lustre: Skipped 5 previous similar messages [877386.225866] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [877386.236136] Lustre: Skipped 30 previous similar messages [878566.555220] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [878566.565488] Lustre: Skipped 7 previous similar messages [879203.622860] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [879203.633136] Lustre: Skipped 1 previous similar message [879896.806456] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [879896.816720] Lustre: Skipped 7 previous similar messages [880559.240252] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [880559.250517] Lustre: Skipped 6 previous similar messages [881698.711798] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [881698.722064] Lustre: Skipped 2 previous similar messages [882636.071880] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [882636.082139] Lustre: Skipped 8 previous similar messages [883389.974721] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [883389.984980] Lustre: Skipped 1 previous similar message [884142.245345] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [884142.255611] Lustre: Skipped 9 previous similar messages [884878.834587] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [884878.844873] Lustre: Skipped 7 previous similar messages [885831.365204] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [885831.375770] Lustre: Skipped 3 previous similar messages [886629.130014] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [886629.140540] Lustre: Skipped 14 previous similar messages [887828.278092] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [887828.288437] Lustre: Skipped 6 previous similar messages [888692.232624] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [888692.242976] Lustre: Skipped 5 previous similar messages [889430.852671] Lustre: fir-OST0003: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [889430.863020] Lustre: Skipped 1 previous similar message [890336.941548] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [890336.952243] Lustre: Skipped 12 previous similar messages [891189.053847] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [891189.064200] Lustre: Skipped 4 previous similar messages [892168.636113] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [892836.346700] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [892836.357051] Lustre: Skipped 5 previous similar messages [893532.669055] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [894501.569533] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [894501.579883] Lustre: Skipped 9 previous similar messages [896202.183256] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [896202.193610] Lustre: Skipped 5 previous similar messages [897414.248361] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [897917.186780] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [897917.197205] Lustre: Skipped 5 previous similar messages [898862.613574] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [898875.613981] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [899043.214711] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [899633.876889] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [899633.887246] Lustre: Skipped 3 previous similar messages [899776.390699] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [900577.852586] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [901330.499379] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [901330.509733] Lustre: Skipped 5 previous similar messages [901417.590705] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [901847.665334] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [901938.667377] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [901938.677749] Lustre: Skipped 5 previous similar messages [902139.385288] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [902139.395656] Lustre: Skipped 5 previous similar messages [902232.891367] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [902232.901756] Lustre: Skipped 5 previous similar messages [902312.314271] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [902312.324620] Lustre: Skipped 5 previous similar messages [902455.989494] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [902455.999768] Lustre: Skipped 5 previous similar messages [902802.276487] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [902802.286747] Lustre: Skipped 1 previous similar message [903042.577599] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [903042.587953] Lustre: Skipped 6 previous similar messages [903516.457168] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [903516.467489] Lustre: Skipped 3 previous similar messages [904725.661648] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [904725.671998] Lustre: Skipped 6 previous similar messages [905916.050334] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [906387.247343] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [906387.257696] Lustre: Skipped 6 previous similar messages [908111.105171] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [908111.115521] Lustre: Skipped 5 previous similar messages [909000.547093] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [909000.557445] Lustre: Skipped 5 previous similar messages [909873.706364] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [909873.716715] Lustre: Skipped 5 previous similar messages [911409.594659] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911416.908803] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911453.240116] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [911474.817339] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911481.895072] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911550.110105] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [911550.120449] Lustre: Skipped 5 previous similar messages [911783.030024] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911823.903400] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [911823.913662] Lustre: Skipped 1 previous similar message [911912.274967] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911912.285490] Lustre: Skipped 3 previous similar messages [912156.123178] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [912447.639357] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [912447.649881] Lustre: Skipped 9 previous similar messages [913037.662995] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [913037.673521] Lustre: Skipped 13 previous similar messages [913649.110767] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [913649.121289] Lustre: Skipped 12 previous similar messages [914262.231594] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [914262.242121] Lustre: Skipped 19 previous similar messages [914882.255891] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [914882.266412] Lustre: Skipped 22 previous similar messages [915749.548354] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [915749.558621] Lustre: Skipped 10 previous similar messages [917462.413514] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [917462.423892] Lustre: Skipped 11 previous similar messages [917563.522941] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [917563.533470] Lustre: Skipped 1 previous similar message [917758.287937] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [917758.298460] Lustre: Skipped 8 previous similar messages [918146.886799] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [918146.897321] Lustre: Skipped 7 previous similar messages [918918.046580] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [918918.056852] Lustre: Skipped 11 previous similar messages [919532.951370] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [919532.961637] Lustre: Skipped 8 previous similar messages [920240.670671] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [920240.680932] Lustre: Skipped 10 previous similar messages [920844.716920] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [920844.727177] Lustre: Skipped 2 previous similar messages [921605.477848] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [921605.488202] Lustre: Skipped 9 previous similar messages [922269.105865] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [922269.116127] Lustre: Skipped 3 previous similar messages [922924.395467] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [922924.405732] Lustre: Skipped 3 previous similar messages [923594.809444] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [923594.819703] Lustre: Skipped 8 previous similar messages [924279.289536] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [924279.299798] Lustre: Skipped 1 previous similar message [925047.533106] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [925047.543364] Lustre: Skipped 3 previous similar messages [925767.447707] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [925767.457982] Lustre: Skipped 9 previous similar messages [926375.983281] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [926375.993800] Lustre: Skipped 20 previous similar messages [927049.119610] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [927049.129873] Lustre: Skipped 4 previous similar messages [927818.485170] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [927818.495432] Lustre: Skipped 9 previous similar messages [928425.939198] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [928425.949457] Lustre: Skipped 2 previous similar messages [929317.009232] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [929317.019529] Lustre: Skipped 8 previous similar messages [930216.266856] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [930216.277116] Lustre: Skipped 2 previous similar messages [930842.436792] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [930842.447052] Lustre: Skipped 8 previous similar messages [931557.022410] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [931557.032674] Lustre: Skipped 2 previous similar messages [932221.427788] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [932221.438148] Lustre: Skipped 7 previous similar messages [932835.037361] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [932835.047887] Lustre: Skipped 5 previous similar messages [933562.841681] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [933562.852209] Lustre: Skipped 4 previous similar messages [934201.861024] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [934201.871288] Lustre: Skipped 16 previous similar messages [935042.286084] Lustre: fir-OST000b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [935042.296613] Lustre: Skipped 3 previous similar messages [935698.144261] Lustre: fir-OST0003: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [935698.154620] Lustre: Skipped 9 previous similar messages [936508.833487] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [936508.844009] Lustre: Skipped 2 previous similar messages [937255.491859] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [937255.502121] Lustre: Skipped 11 previous similar messages [937940.477048] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [937940.487313] Lustre: Skipped 2 previous similar messages [938595.424000] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [938595.434263] Lustre: Skipped 8 previous similar messages [939437.309594] Lustre: fir-OST0007: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [939437.319952] Lustre: Skipped 1 previous similar message [939523.227058] LNetError: 79895:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [939523.237319] LNetError: 79895:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.3@o2ib7 (106): c: 8, oc: 0, rc: 8 [940198.796760] Lustre: fir-OST000b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [940198.807106] Lustre: Skipped 12 previous similar messages [940822.795458] Lustre: fir-OST0001: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [940822.805722] Lustre: Skipped 15 previous similar messages [942338.147375] Lustre: fir-OST000b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [942338.157728] Lustre: Skipped 17 previous similar messages [942466.635607] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [942466.645965] Lustre: Skipped 2 previous similar messages [943288.331293] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [943288.341674] Lustre: Skipped 2 previous similar messages [943395.699094] Lustre: fir-OST0001: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [943395.709467] Lustre: Skipped 5 previous similar messages [943473.816459] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [943473.826811] Lustre: Skipped 1 previous similar message [943548.963011] Lustre: fir-OST0009: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [943548.973365] Lustre: Skipped 1 previous similar message [943708.426257] Lustre: fir-OST0007: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [943708.436639] Lustre: Skipped 13 previous similar messages [944057.996234] Lustre: fir-OST0009: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [944058.006589] Lustre: Skipped 19 previous similar messages [944684.179144] Lustre: fir-OST0007: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [944684.190077] Lustre: Skipped 40 previous similar messages [945292.462669] Lustre: fir-OST0009: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [945292.472951] Lustre: Skipped 26 previous similar messages [945988.383054] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [945988.393402] Lustre: Skipped 15 previous similar messages [946588.415435] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [946588.425833] Lustre: Skipped 12 previous similar messages [947189.431327] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [947189.441709] Lustre: Skipped 40 previous similar messages [947794.771353] Lustre: fir-OST000b: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [947794.781696] Lustre: Skipped 66 previous similar messages [948397.116743] Lustre: fir-OST000b: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [948397.127095] Lustre: Skipped 53 previous similar messages [949030.395618] Lustre: fir-OST0009: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [949030.405970] Lustre: Skipped 70 previous similar messages [949888.485065] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [949888.495411] Lustre: Skipped 52 previous similar messages [950488.516283] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [950488.526633] Lustre: Skipped 12 previous similar messages [951088.532544] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [951088.542897] Lustre: Skipped 12 previous similar messages [951688.549095] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [951688.559460] Lustre: Skipped 13 previous similar messages [952325.762093] Lustre: fir-OST0005: Connection restored to (at 10.8.9.6@o2ib6) [952325.769240] Lustre: Skipped 33 previous similar messages [953188.585211] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [953188.595561] Lustre: Skipped 16 previous similar messages [953788.602536] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [953788.612887] Lustre: Skipped 14 previous similar messages [954388.618930] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [954388.629275] Lustre: Skipped 12 previous similar messages [955289.639877] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [955289.650260] Lustre: Skipped 13 previous similar messages [955889.655961] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [955889.666316] Lustre: Skipped 12 previous similar messages [956489.672421] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [956489.682768] Lustre: Skipped 13 previous similar messages [957389.693292] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [957389.703643] Lustre: Skipped 14 previous similar messages [958288.717203] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [958288.727549] Lustre: Skipped 17 previous similar messages [958888.732995] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [958888.743358] Lustre: Skipped 11 previous similar messages [959488.749246] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [959488.759744] Lustre: Skipped 14 previous similar messages [960089.763311] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [960089.773666] Lustre: Skipped 8 previous similar messages [960988.786039] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [960988.796396] Lustre: Skipped 17 previous similar messages [961588.802948] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [961588.813314] Lustre: Skipped 13 previous similar messages [961844.218919] Lustre: fir-OST0005: haven't heard from client 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90bd4ce7dc00, cur 1548739579 expire 1548739429 last 1548739352 [962488.825113] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [962488.835459] Lustre: Skipped 15 previous similar messages [963088.841256] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [963088.851622] Lustre: Skipped 14 previous similar messages [963688.856898] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [963688.867294] Lustre: Skipped 11 previous similar messages [964289.870283] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [964289.880638] Lustre: Skipped 8 previous similar messages [965188.892793] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [965188.903144] Lustre: Skipped 17 previous similar messages [965788.909330] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [965788.919676] Lustre: Skipped 11 previous similar messages [966388.925965] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [966388.936314] Lustre: Skipped 20 previous similar messages [966988.941491] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [966988.951862] Lustre: Skipped 17 previous similar messages [967589.954584] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [967589.964964] Lustre: Skipped 8 previous similar messages [968488.977901] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [968488.988290] Lustre: Skipped 17 previous similar messages [969088.993508] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [969089.003915] Lustre: Skipped 11 previous similar messages [969689.009126] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [969689.019476] Lustre: Skipped 11 previous similar messages [970289.367159] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [970289.377514] Lustre: Skipped 11 previous similar messages [970890.039327] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [970890.049675] Lustre: Skipped 11 previous similar messages [971490.055559] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [971490.066108] Lustre: Skipped 13 previous similar messages [972389.078265] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [972389.088611] Lustre: Skipped 15 previous similar messages [972990.093177] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [972990.103590] Lustre: Skipped 11 previous similar messages [973890.119238] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [973890.129613] Lustre: Skipped 17 previous similar messages [974789.138830] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [974789.149182] Lustre: Skipped 17 previous similar messages [975389.155809] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [975389.166161] Lustre: Skipped 13 previous similar messages [975989.172007] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [975989.182357] Lustre: Skipped 13 previous similar messages [976589.187859] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [976589.198210] Lustre: Skipped 11 previous similar messages [977189.204577] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [977189.214958] Lustre: Skipped 11 previous similar messages [978089.224394] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [978089.234747] Lustre: Skipped 13 previous similar messages [978689.240793] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [978689.251146] Lustre: Skipped 14 previous similar messages [979289.256614] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [979289.266965] Lustre: Skipped 10 previous similar messages [980189.262866] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [980189.273277] Lustre: Skipped 15 previous similar messages [980789.278509] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [980789.288856] Lustre: Skipped 16 previous similar messages [981390.293351] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [981390.303698] Lustre: Skipped 10 previous similar messages [982290.318481] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [982290.328833] Lustre: Skipped 22 previous similar messages [982890.334122] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [982890.344481] Lustre: Skipped 12 previous similar messages [983505.151509] Lustre: fir-OST0001: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [983505.161772] Lustre: Skipped 16 previous similar messages [984389.373282] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [984389.383645] Lustre: Skipped 17 previous similar messages [984989.388489] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [984989.398851] Lustre: Skipped 20 previous similar messages [985589.403956] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [985589.414362] Lustre: Skipped 14 previous similar messages [986189.617495] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [986189.627848] Lustre: Skipped 17 previous similar messages [986973.828717] Lustre: fir-OST0005: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [986973.839068] Lustre: Skipped 27 previous similar messages [987690.459831] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [987690.470183] Lustre: Skipped 18 previous similar messages [988290.474856] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [988290.485234] Lustre: Skipped 13 previous similar messages [988890.491991] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [988890.502375] Lustre: Skipped 11 previous similar messages [989490.507504] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [989490.517848] Lustre: Skipped 12 previous similar messages [990212.338517] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [990212.348866] Lustre: Skipped 18 previous similar messages [990989.547081] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [990989.557536] Lustre: Skipped 17 previous similar messages [991589.563728] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [991589.574082] Lustre: Skipped 13 previous similar messages [992189.580487] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [992189.590834] Lustre: Skipped 15 previous similar messages [992789.595432] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [992789.605815] Lustre: Skipped 12 previous similar messages [993389.626533] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [993389.636908] Lustre: Skipped 7 previous similar messages [993989.643588] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [993989.653939] Lustre: Skipped 15 previous similar messages [994589.660009] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [994589.670356] Lustre: Skipped 12 previous similar messages [995189.676932] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [995189.687298] Lustre: Skipped 11 previous similar messages [995790.688162] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [995790.698711] Lustre: Skipped 6 previous similar messages [996689.712044] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [996689.722514] Lustre: Skipped 17 previous similar messages [997290.726907] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [997290.737263] Lustre: Skipped 11 previous similar messages [997890.743683] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [997890.754074] Lustre: Skipped 12 previous similar messages [998790.765723] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [998790.776093] Lustre: Skipped 16 previous similar messages [999689.789467] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [999689.799884] Lustre: Skipped 17 previous similar messages [1000290.806496] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1000290.816933] Lustre: Skipped 11 previous similar messages [1001189.829618] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1001189.840056] Lustre: Skipped 17 previous similar messages [1001789.845999] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1001789.856442] Lustre: Skipped 12 previous similar messages [1002389.862151] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1002389.872581] Lustre: Skipped 12 previous similar messages [1002989.880091] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1002989.890605] Lustre: Skipped 14 previous similar messages [1003890.885217] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1003890.895653] Lustre: Skipped 15 previous similar messages [1004780.539478] Lustre: fir-OST0009: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1004780.549929] Lustre: Skipped 21 previous similar messages [1005390.923771] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1005390.934215] Lustre: Skipped 14 previous similar messages [1006039.350047] Lustre: fir-OST0005: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1006039.360488] Lustre: Skipped 20 previous similar messages [1006671.726709] Lustre: fir-OST0003: Connection restored to (at 10.8.9.6@o2ib6) [1006671.733936] Lustre: Skipped 36 previous similar messages [1007323.837556] Lustre: fir-OST0009: Connection restored to (at 10.8.9.6@o2ib6) [1007323.844782] Lustre: Skipped 29 previous similar messages [1007976.424915] Lustre: fir-OST0001: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1007976.435348] Lustre: Skipped 35 previous similar messages [1008675.607490] Lustre: fir-OST0003: Connection restored to 75d3df1c-c05f-80f2-4e0d-3991a06b4cf2 (at 10.8.2.9@o2ib6) [1008675.617833] Lustre: Skipped 15 previous similar messages [1009291.024993] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1009291.035630] Lustre: Skipped 22 previous similar messages [1009903.371593] Lustre: fir-OST0007: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1009903.382033] Lustre: Skipped 26 previous similar messages [1010517.387380] Lustre: fir-OST0007: Connection restored to 75d3df1c-c05f-80f2-4e0d-3991a06b4cf2 (at 10.8.2.9@o2ib6) [1010517.397796] Lustre: Skipped 22 previous similar messages [1011330.687729] Lustre: fir-OST0003: Connection restored to 75d3df1c-c05f-80f2-4e0d-3991a06b4cf2 (at 10.8.2.9@o2ib6) [1011330.698090] Lustre: Skipped 19 previous similar messages [1011991.095699] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1011991.106179] Lustre: Skipped 17 previous similar messages [1012060.091164] Lustre: fir-OST0009: Client 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) reconnecting [1012060.101512] Lustre: Skipped 1 previous similar message [1012060.962680] LustreError: 90084:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff90abf9f75450 x1623602976159568/t0(0) o4->28764730-adc8-1b1a-0aa7-9bb3363d4889@10.9.101.59@o2ib4:182/0 lens 504/448 e 1 to 0 dl 1548789817 ref 1 fl Interpret:/0/0 rc 0/0 [1012062.341344] Lustre: fir-OST000b: Client a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2 (at 10.9.101.60@o2ib4) reconnecting [1012063.373293] Lustre: fir-OST0007: Client a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2 (at 10.9.101.60@o2ib4) reconnecting [1012063.383661] Lustre: Skipped 4 previous similar messages [1012063.919683] LNetError: 79906:0:(lib-msg.c:811:lnet_is_health_check()) Msg is in inconsistent state, don't perform health checking (-125, 0) [1012063.932377] LustreError: 79906:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012063.943351] LustreError: 79906:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012063.954343] LustreError: 79906:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012063.965301] LustreError: 79905:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012063.976274] LustreError: 79906:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012066.083807] LNetError: 79895:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 1 seconds [1012066.093981] LNetError: 79895:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.209@o2ib7 (6): c: 0, oc: 3, rc: 8 [1012066.172609] Lustre: fir-OST0001: Client 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) reconnecting [1012066.182955] Lustre: Skipped 1 previous similar message [1012068.893400] LNetError: 79911:0:(lib-msg.c:811:lnet_is_health_check()) Msg is in inconsistent state, don't perform health checking (-125, 0) [1012068.893402] LustreError: 79909:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.893407] LustreError: 79910:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.893413] LustreError: 79912:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.893416] LustreError: 79909:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.924012] LustreError: 79898:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.924045] LustreError: 79900:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.924049] LustreError: 79898:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.924053] LustreError: 79900:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012068.993684] LNetError: 79911:0:(lib-msg.c:811:lnet_is_health_check()) Skipped 11 previous similar messages [1012069.003519] LustreError: 79911:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012070.482956] LNetError: 79903:0:(lib-msg.c:811:lnet_is_health_check()) Msg is in inconsistent state, don't perform health checking (-125, 0) [1012070.495657] LustreError: 79903:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012070.506646] LustreError: 79904:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff909bdd72c600 [1012070.518245] Lustre: fir-OST0009: Bulk IO write error with 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4), client will retry: rc = -110 [1012083.396885] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012083.414462] LustreError: Skipped 3 previous similar messages [1012083.650451] Lustre: fir-OST0001: Client 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) reconnecting [1012083.650452] Lustre: fir-OST0003: Client 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) reconnecting [1012083.650455] Lustre: Skipped 6 previous similar messages [1012083.676063] Lustre: Skipped 4 previous similar messages [1012094.600243] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012094.617525] LustreError: Skipped 1 previous similar message [1012105.615426] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.9.0.1@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012105.632766] LustreError: Skipped 5 previous similar messages [1012108.448630] Lustre: fir-OST0009: Client a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2 (at 10.9.101.60@o2ib4) reconnecting [1012108.458988] Lustre: Skipped 1 previous similar message [1012155.919109] Lustre: 5137:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548789882/real 1548789882] req@ffff90be2ec35d00 x1623662709502928/t0(0) o104->fir-OST000b@10.9.0.1@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1548789889 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1012162.946291] Lustre: 5137:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548789889/real 1548789889] req@ffff90be2ec35d00 x1623662709502928/t0(0) o104->fir-OST000b@10.9.0.1@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1548789896 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1012169.973469] Lustre: 5137:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548789896/real 1548789896] req@ffff90be2ec35d00 x1623662709502928/t0(0) o104->fir-OST000b@10.9.0.1@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1548789903 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1012172.401920] Lustre: fir-OST0005: Client 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) reconnecting [1012172.412008] Lustre: Skipped 10 previous similar messages [1012185.353719] LustreError: 137-5: fir-OST0006_UUID: not available for connect from 10.9.0.1@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012185.371034] LustreError: Skipped 2 previous similar messages [1012196.179137] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.9.0.1@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012791.310108] Lustre: fir-OST0005: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1012791.320568] Lustre: Skipped 71 previous similar messages [1013501.070298] Lustre: fir-OST0007: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1013501.080734] Lustre: Skipped 23 previous similar messages [1014230.955959] Lustre: fir-OST0007: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1014230.966401] Lustre: Skipped 13 previous similar messages [1014846.601978] Lustre: fir-OST0007: Connection restored to 75d3df1c-c05f-80f2-4e0d-3991a06b4cf2 (at 10.8.2.9@o2ib6) [1014846.612328] Lustre: Skipped 16 previous similar messages [1015495.602508] Lustre: fir-OST0009: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1015495.612896] Lustre: Skipped 6 previous similar messages [1016135.709356] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1016135.719979] Lustre: Skipped 63 previous similar messages [1016736.960476] Lustre: fir-OST000b: Connection restored to (at 10.8.9.6@o2ib6) [1016736.967708] Lustre: Skipped 54 previous similar messages [1017351.675150] Lustre: fir-OST0009: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1017351.685590] Lustre: Skipped 47 previous similar messages [1017990.280438] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1017990.290926] Lustre: Skipped 41 previous similar messages [1018890.276262] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1018890.286700] Lustre: Skipped 26 previous similar messages [1019491.291875] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1019491.302312] Lustre: Skipped 54 previous similar messages [1020098.258524] Lustre: fir-OST000b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1020098.269068] Lustre: Skipped 63 previous similar messages [1020747.248284] Lustre: fir-OST0001: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1020747.258720] Lustre: Skipped 75 previous similar messages [1021393.505244] Lustre: fir-OST0003: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1021393.515680] Lustre: Skipped 44 previous similar messages [1022104.602102] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1022104.612753] Lustre: Skipped 66 previous similar messages [1022791.393860] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1022791.404296] Lustre: Skipped 64 previous similar messages [1023690.416656] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1023690.427088] Lustre: Skipped 20 previous similar messages [1024290.434592] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1024290.445029] Lustre: Skipped 12 previous similar messages [1024890.450596] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1024890.461051] Lustre: Skipped 14 previous similar messages [1025790.473862] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1025790.484311] Lustre: Skipped 26 previous similar messages [1026390.489902] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1026390.500364] Lustre: Skipped 17 previous similar messages [1026990.511631] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1026990.522070] Lustre: Skipped 9 previous similar messages [1027695.392666] Lustre: fir-OST0005: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1027695.403233] Lustre: Skipped 20 previous similar messages [1028440.597294] Lustre: fir-OST0009: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1028440.607726] Lustre: Skipped 27 previous similar messages [1029090.557095] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1029090.567549] Lustre: Skipped 18 previous similar messages [1029690.573463] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1029690.583896] Lustre: Skipped 19 previous similar messages [1030291.588524] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1030291.598996] Lustre: Skipped 15 previous similar messages [1030891.605989] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1030891.616428] Lustre: Skipped 15 previous similar messages [1032250.789481] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1032250.799921] Lustre: Skipped 18 previous similar messages [1032371.851431] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1033405.493136] Lustre: fir-OST0001: Connection restored to 8ae4cc68-204d-d2bb-4908-94f49692744b (at 10.8.20.15@o2ib6) [1033405.503674] Lustre: Skipped 5 previous similar messages [1037970.258223] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1037988.334895] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1037988.345334] Lustre: Skipped 1 previous similar message [1038058.114816] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1038071.679929] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1038071.690425] Lustre: Skipped 2 previous similar messages [1038109.656797] Lustre: fir-OST0001: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1038109.667235] Lustre: Skipped 1 previous similar message [1038160.903824] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1038160.914187] Lustre: Skipped 7 previous similar messages [1039044.606030] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1039044.616413] Lustre: Skipped 3 previous similar messages [1039439.031355] Lustre: fir-OST0005: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1039439.041972] Lustre: Skipped 2 previous similar messages [1040023.199782] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1040023.210137] Lustre: Skipped 3 previous similar messages [1040215.469107] Lustre: fir-OST0007: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1040215.479716] Lustre: Skipped 1 previous similar message [1040294.982439] Lustre: fir-OST0007: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1040294.993073] Lustre: Skipped 2 previous similar messages [1040578.078630] Lustre: fir-OST0005: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1040578.089253] Lustre: Skipped 2 previous similar messages [1040972.461904] Lustre: fir-OST0007: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1040972.472523] Lustre: Skipped 10 previous similar messages [1041750.626560] Lustre: fir-OST0001: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1041750.637169] Lustre: Skipped 6 previous similar messages [1042690.186560] Lustre: fir-OST000b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1042690.196995] Lustre: Skipped 17 previous similar messages [1043462.933895] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1043462.944244] Lustre: Skipped 24 previous similar messages [1044114.619110] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1044114.629460] Lustre: Skipped 11 previous similar messages [1044802.899666] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1044802.910018] Lustre: Skipped 11 previous similar messages [1045408.202097] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1045408.212463] Lustre: Skipped 13 previous similar messages [1046071.480856] Lustre: fir-OST0001: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1046071.491294] Lustre: Skipped 13 previous similar messages [1046675.472161] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1046675.482576] Lustre: Skipped 26 previous similar messages [1047365.986434] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1047365.996783] Lustre: Skipped 9 previous similar messages [1048011.634241] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1048011.644641] Lustre: Skipped 5 previous similar messages [1049156.749550] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1049156.759908] Lustre: Skipped 3 previous similar messages [1049859.352823] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1049859.363170] Lustre: Skipped 3 previous similar messages [1050713.648046] Lustre: fir-OST0005: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1050713.658654] Lustre: Skipped 3 previous similar messages [1051397.583784] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1051397.594133] Lustre: Skipped 23 previous similar messages [1052017.820149] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1052017.830504] Lustre: Skipped 9 previous similar messages [1052646.971933] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1052646.982284] Lustre: Skipped 16 previous similar messages [1052923.234782] perf: interrupt took too long (3917 > 3908), lowering kernel.perf_event_max_sample_rate to 51000 [1053322.881942] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1053322.892284] Lustre: Skipped 13 previous similar messages [1054201.391195] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1054201.401546] Lustre: Skipped 7 previous similar messages [1055560.051786] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1055560.062135] Lustre: Skipped 5 previous similar messages [1055763.964550] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1055763.974901] Lustre: Skipped 1 previous similar message [1055986.413591] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1055986.423941] Lustre: Skipped 5 previous similar messages [1058000.118640] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1058000.128986] Lustre: Skipped 1 previous similar message [1058464.485156] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1058464.495511] Lustre: Skipped 1 previous similar message [1058673.436768] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1058673.447112] Lustre: Skipped 2 previous similar messages [1059516.362315] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1059516.372665] Lustre: Skipped 2 previous similar messages [1059569.485476] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1059569.495826] Lustre: Skipped 1 previous similar message [1059676.162232] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1059676.172579] Lustre: Skipped 1 previous similar message [1059919.362490] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1059919.372842] Lustre: Skipped 1 previous similar message [1060125.056319] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1060125.066935] Lustre: Skipped 1 previous similar message [1060472.060872] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1060472.071233] Lustre: Skipped 11 previous similar messages [1061367.816043] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1061367.826389] Lustre: Skipped 1 previous similar message [1062078.687452] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1062078.697803] Lustre: Skipped 9 previous similar messages [1062679.029703] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1062679.040054] Lustre: Skipped 5 previous similar messages [1063712.282735] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1063712.293085] Lustre: Skipped 4 previous similar messages [1064402.010968] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1064402.021316] Lustre: Skipped 11 previous similar messages [1065005.730026] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1065005.740386] Lustre: Skipped 9 previous similar messages [1065707.489158] Lustre: fir-OST0007: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1065707.499513] Lustre: Skipped 23 previous similar messages [1066449.499868] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1066449.510221] Lustre: Skipped 22 previous similar messages [1067121.263831] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1067121.274182] Lustre: Skipped 10 previous similar messages [1068072.242641] Lustre: fir-OST0001: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1068072.252996] Lustre: Skipped 5 previous similar messages [1068910.772241] Lustre: fir-OST0003: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1068910.782689] Lustre: Skipped 1 previous similar message [1069710.386260] Lustre: fir-OST0001: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1070321.816345] Lustre: fir-OST000b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1070321.826793] Lustre: Skipped 24 previous similar messages [1070925.173087] Lustre: fir-OST000b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1070925.183525] Lustre: Skipped 25 previous similar messages [1072679.261000] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1072679.271647] Lustre: Skipped 10 previous similar messages [1073253.181034] Lustre: fir-OST0009: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1073253.191388] Lustre: Skipped 8 previous similar messages [1073551.389595] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1073551.399943] Lustre: Skipped 2 previous similar messages [1074121.033205] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1074121.043574] Lustre: Skipped 3 previous similar messages [1075083.332410] Lustre: fir-OST0007: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1075083.343022] Lustre: Skipped 7 previous similar messages [1075834.870410] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1075834.880755] Lustre: Skipped 21 previous similar messages [1076524.122274] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1076524.132622] Lustre: Skipped 7 previous similar messages [1077188.001436] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1077188.011789] Lustre: Skipped 6 previous similar messages [1080777.181881] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1080777.192234] Lustre: Skipped 7 previous similar messages [1080858.062109] Lustre: fir-OST0007: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1080858.072459] Lustre: Skipped 4 previous similar messages [1081014.594651] Lustre: fir-OST0007: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1081014.605012] Lustre: Skipped 15 previous similar messages [1081346.616486] Lustre: fir-OST0001: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1081346.626969] Lustre: Skipped 4 previous similar messages [1082149.121001] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1082149.131349] Lustre: Skipped 37 previous similar messages [1082750.828849] Lustre: fir-OST0005: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1082750.839293] Lustre: Skipped 21 previous similar messages [1083362.474966] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1083362.485409] Lustre: Skipped 45 previous similar messages [1083966.770485] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1083966.781117] Lustre: Skipped 21 previous similar messages [1084607.727202] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1084607.737639] Lustre: Skipped 24 previous similar messages [1085208.919021] Lustre: fir-OST0005: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1085208.929628] Lustre: Skipped 23 previous similar messages [1085892.308430] Lustre: fir-OST0005: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1085892.318777] Lustre: Skipped 20 previous similar messages [1086599.439389] Lustre: fir-OST0003: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1086599.449833] Lustre: Skipped 20 previous similar messages [1087284.320201] Lustre: fir-OST0009: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1087284.330814] Lustre: Skipped 12 previous similar messages [1087906.883014] Lustre: fir-OST0003: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1087906.893629] Lustre: Skipped 14 previous similar messages [1088522.246628] Lustre: fir-OST0009: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1088522.257065] Lustre: Skipped 17 previous similar messages [1089138.004850] Lustre: fir-OST0007: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1089138.015283] Lustre: Skipped 41 previous similar messages [1096026.192363] Lustre: fir-OST0001: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1096026.202710] Lustre: Skipped 17 previous similar messages [1098566.024168] Lustre: fir-OST0007: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1098566.034721] Lustre: Skipped 14 previous similar messages [1098615.625084] Lustre: fir-OST000b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1098635.323440] Lustre: fir-OST0007: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1098635.333882] Lustre: Skipped 1 previous similar message [1098763.140214] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1098763.150564] Lustre: Skipped 4 previous similar messages [1098850.867030] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1098850.877391] Lustre: Skipped 3 previous similar messages [1099052.950223] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1099052.960574] Lustre: Skipped 3 previous similar messages [1099373.444160] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1099373.454523] Lustre: Skipped 11 previous similar messages [1100095.761071] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1100095.771422] Lustre: Skipped 15 previous similar messages [1100779.619554] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1100779.629901] Lustre: Skipped 10 previous similar messages [1101563.846439] Lustre: fir-OST0005: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1101563.856896] Lustre: Skipped 8 previous similar messages [1106592.746189] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1106592.756622] Lustre: Skipped 8 previous similar messages [1119673.443491] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1119673.453840] Lustre: Skipped 4 previous similar messages [1119799.607563] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1119799.618104] Lustre: Skipped 1 previous similar message [1121234.352379] Lustre: fir-OST0009: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1121234.362815] Lustre: Skipped 4 previous similar messages [1149840.503418] md: md1: data-check done. [1151756.529752] md: md11: data-check done. [1157872.886377] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1157874.300943] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1174689.823015] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1174689.833459] Lustre: Skipped 2 previous similar messages [1174711.857087] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1178878.159960] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1179015.372103] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1179015.382542] Lustre: Skipped 7 previous similar messages [1179039.549206] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1179108.867034] Lustre: fir-OST0009: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1179108.877473] Lustre: Skipped 2 previous similar messages [1179422.712216] Lustre: fir-OST0005: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1180497.603727] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1180497.614163] Lustre: Skipped 5 previous similar messages [1180508.405178] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1180508.415609] Lustre: Skipped 3 previous similar messages [1180617.990349] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1180669.115420] Lustre: fir-OST000b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1180669.125776] Lustre: Skipped 3 previous similar messages [1183931.730586] Lustre: fir-OST0005: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1184048.119950] Lustre: fir-OST0007: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1184048.130395] Lustre: Skipped 1 previous similar message [1184052.060133] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1186133.508340] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1186133.518808] Lustre: Skipped 6 previous similar messages [1186190.706699] Lustre: fir-OST0003: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1186660.891518] Lustre: fir-OST000b: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1186660.901955] Lustre: Skipped 2 previous similar messages [1186832.222721] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1186832.222722] Lustre: fir-OST0007: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187109.575856] Lustre: fir-OST0007: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187109.575857] Lustre: fir-OST0001: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187320.902820] Lustre: fir-OST0007: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187320.913281] Lustre: Skipped 1 previous similar message [1187542.233407] Lustre: fir-OST0007: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187542.243849] Lustre: Skipped 1 previous similar message [1187923.312648] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1188212.799459] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1188212.809891] Lustre: Skipped 2 previous similar messages [1188753.428376] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1189328.626019] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1189328.636470] Lustre: Skipped 18 previous similar messages [1189931.416447] md: md7: data-check done. [1191095.752461] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1191095.762904] Lustre: Skipped 3 previous similar messages [1191180.179740] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1191180.190204] Lustre: Skipped 2 previous similar messages [1191330.383591] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1191330.394044] Lustre: Skipped 1 previous similar message [1191694.767194] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1191694.777659] Lustre: Skipped 6 previous similar messages [1192360.518246] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1192360.528692] Lustre: Skipped 9 previous similar messages [1193035.609208] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1193035.619562] Lustre: Skipped 12 previous similar messages [1193655.506973] Lustre: fir-OST0007: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1193655.517424] Lustre: Skipped 8 previous similar messages [1194268.696398] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1194268.706833] Lustre: Skipped 2 previous similar messages [1199731.978045] md: md3: data-check done. [1209861.795967] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1209861.806407] Lustre: Skipped 6 previous similar messages [1210440.108271] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1211365.035865] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1211365.046334] Lustre: Skipped 3 previous similar messages [1211477.831965] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1211477.842450] Lustre: Skipped 3 previous similar messages [1212162.620520] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1213974.751857] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1216460.448686] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1216460.459116] Lustre: Skipped 2 previous similar messages [1216895.410627] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1217196.420157] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1217196.430588] Lustre: Skipped 1 previous similar message [1217495.428380] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1217495.438821] Lustre: Skipped 1 previous similar message [1217634.172363] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1217634.182830] Lustre: Skipped 1 previous similar message [1219288.939532] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1219288.949965] Lustre: Skipped 4 previous similar messages [1219290.356629] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1219290.367065] Lustre: Skipped 1 previous similar message [1219596.491590] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1219896.506494] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1219896.516926] Lustre: Skipped 5 previous similar messages [1220196.514872] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1220196.525307] Lustre: Skipped 5 previous similar messages [1220495.523150] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1220495.533596] Lustre: Skipped 5 previous similar messages [1220795.531872] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1220795.542320] Lustre: Skipped 5 previous similar messages [1221095.537529] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1221095.547980] Lustre: Skipped 5 previous similar messages [1221395.544613] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1221395.555045] Lustre: Skipped 5 previous similar messages [1221695.562635] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1221695.573109] Lustre: Skipped 5 previous similar messages [1222296.567986] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1222296.578420] Lustre: Skipped 11 previous similar messages [1223195.591423] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1223195.601857] Lustre: Skipped 17 previous similar messages [1223795.609019] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1223795.619461] Lustre: Skipped 25 previous similar messages [1224395.625747] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1224395.636184] Lustre: Skipped 21 previous similar messages [1224995.641892] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1224995.652321] Lustre: Skipped 11 previous similar messages [1225595.658614] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1225595.669051] Lustre: Skipped 12 previous similar messages [1226495.676644] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1226495.687078] Lustre: Skipped 12 previous similar messages [1227096.692563] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1227096.702996] Lustre: Skipped 11 previous similar messages [1227995.714972] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1227995.725426] Lustre: Skipped 17 previous similar messages [1228596.730094] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1228596.740539] Lustre: Skipped 11 previous similar messages [1229495.755326] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1229495.765766] Lustre: Skipped 17 previous similar messages [1230395.775388] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1230395.785824] Lustre: Skipped 17 previous similar messages [1230995.792216] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1230995.802650] Lustre: Skipped 12 previous similar messages [1231595.807955] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1231595.818396] Lustre: Skipped 11 previous similar messages [1232195.824327] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1232195.834761] Lustre: Skipped 12 previous similar messages [1233096.844879] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1233096.855318] Lustre: Skipped 15 previous similar messages [1233995.867797] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1233995.878237] Lustre: Skipped 17 previous similar messages [1234595.885177] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1234595.895628] Lustre: Skipped 11 previous similar messages [1235195.901021] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1235195.911459] Lustre: Skipped 14 previous similar messages [1236095.904746] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1236095.915185] Lustre: Skipped 14 previous similar messages [1236696.937269] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1236696.947699] Lustre: Skipped 11 previous similar messages [1237596.957911] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1237596.968348] Lustre: Skipped 17 previous similar messages [1238196.975106] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1238196.985544] Lustre: Skipped 12 previous similar messages [1238796.991649] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1238797.002106] Lustre: Skipped 13 previous similar messages [1239696.013531] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1239696.023971] Lustre: Skipped 14 previous similar messages [1240296.030966] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1240296.041413] Lustre: Skipped 13 previous similar messages [1240896.046578] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1240896.057016] Lustre: Skipped 12 previous similar messages [1241796.067742] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1241796.078180] Lustre: Skipped 14 previous similar messages [1242396.085210] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1242396.095646] Lustre: Skipped 12 previous similar messages [1242996.101944] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1242996.112390] Lustre: Skipped 14 previous similar messages [1243896.122168] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1243896.132610] Lustre: Skipped 13 previous similar messages [1244496.138435] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1244496.148867] Lustre: Skipped 14 previous similar messages [1245096.155351] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1245096.165785] Lustre: Skipped 10 previous similar messages [1245696.171111] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1245696.181548] Lustre: Skipped 10 previous similar messages [1246296.201255] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1246296.211690] Lustre: Skipped 10 previous similar messages [1246897.199462] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1246897.209899] Lustre: Skipped 11 previous similar messages [1247796.223760] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1247796.234196] Lustre: Skipped 17 previous similar messages [1248397.237544] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1248397.247985] Lustre: Skipped 11 previous similar messages [1248997.254150] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1248997.264600] Lustre: Skipped 11 previous similar messages [1249897.274889] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1249897.285324] Lustre: Skipped 17 previous similar messages [1250796.301110] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1250796.311567] Lustre: Skipped 17 previous similar messages [1251696.323881] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1251696.334324] Lustre: Skipped 17 previous similar messages [1252296.340097] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1252296.350539] Lustre: Skipped 13 previous similar messages [1252897.353323] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1252897.363762] Lustre: Skipped 9 previous similar messages [1253796.387208] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1253796.397838] Lustre: Skipped 17 previous similar messages [1254696.400117] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1254696.410552] Lustre: Skipped 17 previous similar messages [1255296.416916] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1255296.427353] Lustre: Skipped 12 previous similar messages [1256196.436939] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1256196.447387] Lustre: Skipped 16 previous similar messages [1256796.453618] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1256796.464058] Lustre: Skipped 11 previous similar messages [1257396.470300] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1257396.480739] Lustre: Skipped 11 previous similar messages [1257996.485834] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1257996.496275] Lustre: Skipped 13 previous similar messages [1258597.498569] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1258597.509009] Lustre: Skipped 9 previous similar messages [1259496.522223] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1259496.532744] Lustre: Skipped 17 previous similar messages [1260096.538279] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1260096.548720] Lustre: Skipped 12 previous similar messages [1260696.555660] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1260696.566122] Lustre: Skipped 13 previous similar messages [1261597.560737] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1261597.571175] Lustre: Skipped 14 previous similar messages [1262496.585404] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1262496.595848] Lustre: Skipped 17 previous similar messages [1263097.600356] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1263097.610796] Lustre: Skipped 11 previous similar messages [1263996.624387] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1263996.634830] Lustre: Skipped 17 previous similar messages [1264596.640631] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1264596.651261] Lustre: Skipped 13 previous similar messages [1265496.663174] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1265496.673621] Lustre: Skipped 15 previous similar messages [1266096.678996] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1266096.689443] Lustre: Skipped 13 previous similar messages [1266221.291348] md: md9: data-check done. [1266696.694559] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1266696.705003] Lustre: Skipped 12 previous similar messages [1267296.709756] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1267296.720197] Lustre: Skipped 11 previous similar messages [1267896.725545] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1267896.735980] Lustre: Skipped 8 previous similar messages [1268497.740118] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1268497.750549] Lustre: Skipped 11 previous similar messages [1269397.762162] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1269397.772596] Lustre: Skipped 17 previous similar messages [1270296.784706] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1270296.795151] Lustre: Skipped 17 previous similar messages [1270896.801705] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1270896.812137] Lustre: Skipped 11 previous similar messages [1271496.817400] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1271496.827839] Lustre: Skipped 14 previous similar messages [1272096.832340] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1272096.842770] Lustre: Skipped 8 previous similar messages [1272696.896051] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1272696.906486] Lustre: Skipped 11 previous similar messages [1273297.880621] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1273297.891067] Lustre: Skipped 11 previous similar messages [1274196.902825] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1274196.913257] Lustre: Skipped 17 previous similar messages [1274796.921418] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1274796.931895] Lustre: Skipped 11 previous similar messages [1275397.934541] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1275397.944994] Lustre: Skipped 11 previous similar messages [1276297.958533] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1276297.968974] Lustre: Skipped 17 previous similar messages [1277196.980774] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1277196.991249] Lustre: Skipped 17 previous similar messages [1277796.997458] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1277797.007898] Lustre: Skipped 11 previous similar messages [1278397.013819] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1278397.024257] Lustre: Skipped 13 previous similar messages [1278998.026884] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1278998.037374] Lustre: Skipped 9 previous similar messages [1279897.053527] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1279897.063962] Lustre: Skipped 17 previous similar messages [1280497.070072] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1280497.080506] Lustre: Skipped 13 previous similar messages [1281098.081381] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1281098.091822] Lustre: Skipped 9 previous similar messages [1281998.104365] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1281998.114804] Lustre: Skipped 17 previous similar messages [1282897.126199] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1282897.136632] Lustre: Skipped 17 previous similar messages [1283497.143728] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1283497.154173] Lustre: Skipped 34 previous similar messages [1284397.163144] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1284397.173581] Lustre: Skipped 28 previous similar messages [1284997.179961] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1284997.190401] Lustre: Skipped 25 previous similar messages [1285897.202139] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1285897.212585] Lustre: Skipped 21 previous similar messages [1286497.220080] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1286497.230515] Lustre: Skipped 13 previous similar messages [1287098.230377] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1287098.240822] Lustre: Skipped 9 previous similar messages [1287997.253775] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1287997.264219] Lustre: Skipped 17 previous similar messages [1288597.271217] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1288597.281652] Lustre: Skipped 13 previous similar messages [1289198.284150] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1289198.294586] Lustre: Skipped 9 previous similar messages [1290097.308308] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1290097.318748] Lustre: Skipped 17 previous similar messages [1290697.326519] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1290697.336957] Lustre: Skipped 12 previous similar messages [1291297.343228] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1291297.353660] Lustre: Skipped 14 previous similar messages [1292197.363621] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1292197.374053] Lustre: Skipped 13 previous similar messages [1292798.379715] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1292798.390379] Lustre: Skipped 11 previous similar messages [1293697.402395] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1293697.412835] Lustre: Skipped 17 previous similar messages [1294298.418222] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1294298.428657] Lustre: Skipped 11 previous similar messages [1295197.440279] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1295197.450723] Lustre: Skipped 17 previous similar messages [1295797.456639] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1295797.467073] Lustre: Skipped 12 previous similar messages [1296397.473344] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1296397.483774] Lustre: Skipped 12 previous similar messages [1296998.489218] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1296998.499656] Lustre: Skipped 9 previous similar messages [1297898.513109] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1297898.523549] Lustre: Skipped 17 previous similar messages [1298797.533856] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1298797.544294] Lustre: Skipped 17 previous similar messages [1299397.551205] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1299397.561638] Lustre: Skipped 11 previous similar messages [1299997.568938] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1299997.579376] Lustre: Skipped 12 previous similar messages [1300897.591537] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1300897.601986] Lustre: Skipped 16 previous similar messages [1301497.610753] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1301497.621188] Lustre: Skipped 16 previous similar messages [1302098.623065] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1302098.633508] Lustre: Skipped 6 previous similar messages [1302997.646618] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1302997.657084] Lustre: Skipped 17 previous similar messages [1303598.661783] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1303598.672222] Lustre: Skipped 11 previous similar messages [1304497.684974] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1304497.695413] Lustre: Skipped 17 previous similar messages [1305397.707645] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1305397.718088] Lustre: Skipped 17 previous similar messages [1306297.730143] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1306297.740594] Lustre: Skipped 17 previous similar messages [1306897.746348] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1306897.756787] Lustre: Skipped 16 previous similar messages [1307498.759622] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1307498.770063] Lustre: Skipped 6 previous similar messages [1308398.780829] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1308398.791265] Lustre: Skipped 17 previous similar messages [1309298.804241] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1309298.814675] Lustre: Skipped 17 previous similar messages [1309898.820821] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1309898.831262] Lustre: Skipped 15 previous similar messages [1310797.844316] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1310797.854756] Lustre: Skipped 13 previous similar messages [1311397.861862] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1311397.872296] Lustre: Skipped 15 previous similar messages [1311997.877912] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1311997.888348] Lustre: Skipped 12 previous similar messages [1312897.899757] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1312897.910199] Lustre: Skipped 12 previous similar messages [1313498.913325] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1313498.923762] Lustre: Skipped 11 previous similar messages [1314398.936421] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1314398.946864] Lustre: Skipped 17 previous similar messages [1315298.960223] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1315298.970662] Lustre: Skipped 17 previous similar messages [1316198.981239] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1316198.991678] Lustre: Skipped 17 previous similar messages [1317098.003878] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1317098.014315] Lustre: Skipped 17 previous similar messages [1317698.020125] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1317698.030565] Lustre: Skipped 13 previous similar messages [1318298.036673] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1318298.047108] Lustre: Skipped 10 previous similar messages [1319198.056011] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1319198.066447] Lustre: Skipped 16 previous similar messages [1319799.070377] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1319799.080808] Lustre: Skipped 11 previous similar messages [1320699.094595] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1320699.105047] Lustre: Skipped 17 previous similar messages [1321598.118306] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1321598.128744] Lustre: Skipped 17 previous similar messages [1322057.841104] md: md5: data-check done. [1322198.134832] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1322198.145269] Lustre: Skipped 12 previous similar messages [1322798.151867] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1322798.162306] Lustre: Skipped 10 previous similar messages [1323698.174697] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1323698.185135] Lustre: Skipped 17 previous similar messages [1324298.192112] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1324298.202549] Lustre: Skipped 16 previous similar messages [1325198.213510] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1325198.223939] Lustre: Skipped 12 previous similar messages [1325798.230901] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1325798.241347] Lustre: Skipped 14 previous similar messages [1326398.248515] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1326398.258955] Lustre: Skipped 12 previous similar messages [1326999.259982] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1326999.270426] Lustre: Skipped 7 previous similar messages [1327898.284210] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1327898.294650] Lustre: Skipped 17 previous similar messages [1328498.301293] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1328498.311729] Lustre: Skipped 13 previous similar messages [1329099.316751] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1329099.327186] Lustre: Skipped 9 previous similar messages [1329998.339407] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1329998.349839] Lustre: Skipped 17 previous similar messages [1330598.356651] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1330598.367105] Lustre: Skipped 16 previous similar messages [1331199.368167] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1331199.378599] Lustre: Skipped 6 previous similar messages [1332099.394623] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1332099.405061] Lustre: Skipped 17 previous similar messages [1332998.417895] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1332998.428328] Lustre: Skipped 17 previous similar messages [1333599.431701] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1333599.442140] Lustre: Skipped 11 previous similar messages [1334498.455207] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1334498.465646] Lustre: Skipped 17 previous similar messages [1335098.472379] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1335098.482817] Lustre: Skipped 16 previous similar messages [1335698.488330] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1335698.498768] Lustre: Skipped 10 previous similar messages [1336598.509200] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1336598.519634] Lustre: Skipped 13 previous similar messages [1337198.525807] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1337198.536243] Lustre: Skipped 11 previous similar messages [1338099.546955] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1338099.557393] Lustre: Skipped 17 previous similar messages [1338998.571432] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1338998.581868] Lustre: Skipped 17 previous similar messages [1339599.586249] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1339599.596699] Lustre: Skipped 11 previous similar messages [1340498.612528] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1340498.622970] Lustre: Skipped 17 previous similar messages [1341098.629119] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1341098.639555] Lustre: Skipped 14 previous similar messages [1341998.647964] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1341998.658408] Lustre: Skipped 14 previous similar messages [1342598.665657] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1342598.676088] Lustre: Skipped 16 previous similar messages [1343198.681661] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1343198.692096] Lustre: Skipped 10 previous similar messages [1343798.697746] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1343798.708190] Lustre: Skipped 12 previous similar messages [1344398.713645] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1344398.724077] Lustre: Skipped 11 previous similar messages [1345298.735453] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1345298.745894] Lustre: Skipped 12 previous similar messages [1345898.752295] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1345898.762727] Lustre: Skipped 12 previous similar messages [1346799.771624] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1346799.782073] Lustre: Skipped 16 previous similar messages [1347399.788314] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1347399.798749] Lustre: Skipped 11 previous similar messages [1347999.804434] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1347999.814882] Lustre: Skipped 14 previous similar messages [1348599.820835] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1348599.831278] Lustre: Skipped 11 previous similar messages [1349498.843168] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1349498.853600] Lustre: Skipped 14 previous similar messages [1350098.860875] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1350098.871323] Lustre: Skipped 16 previous similar messages [1350998.882520] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1350998.892961] Lustre: Skipped 12 previous similar messages [1351598.898961] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1351598.909421] Lustre: Skipped 14 previous similar messages [1352199.914222] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1352199.924658] Lustre: Skipped 8 previous similar messages [1352799.931278] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1352799.941708] Lustre: Skipped 15 previous similar messages [1353399.948047] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1353399.958483] Lustre: Skipped 11 previous similar messages [1354298.969170] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1354298.979612] Lustre: Skipped 13 previous similar messages [1354898.986528] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1354898.996974] Lustre: Skipped 12 previous similar messages [1355499.003588] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1355499.014039] Lustre: Skipped 13 previous similar messages [1356100.015700] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1356100.026153] Lustre: Skipped 8 previous similar messages [1356999.039692] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1356999.050127] Lustre: Skipped 17 previous similar messages [1357599.056786] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1357599.067221] Lustre: Skipped 13 previous similar messages [1358199.072828] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1358199.083268] Lustre: Skipped 14 previous similar messages [1358800.085606] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1358800.096064] Lustre: Skipped 6 previous similar messages [1359699.110511] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1359699.120952] Lustre: Skipped 17 previous similar messages [1360300.126200] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1360300.136648] Lustre: Skipped 11 previous similar messages [1361200.149353] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1361200.159793] Lustre: Skipped 17 previous similar messages [1362099.174208] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1362099.184647] Lustre: Skipped 17 previous similar messages [1362699.190993] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1362699.201433] Lustre: Skipped 12 previous similar messages [1363600.210294] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1363600.220732] Lustre: Skipped 16 previous similar messages [1364499.235907] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1364499.246351] Lustre: Skipped 17 previous similar messages [1365099.253282] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1365099.263721] Lustre: Skipped 12 previous similar messages [1365699.268886] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1365699.279321] Lustre: Skipped 14 previous similar messages [1366299.284790] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1366299.295249] Lustre: Skipped 10 previous similar messages [1366900.295922] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1366900.306357] Lustre: Skipped 8 previous similar messages [1367500.312701] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1367500.323139] Lustre: Skipped 11 previous similar messages [1368400.334647] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1368400.345082] Lustre: Skipped 17 previous similar messages [1369299.359191] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1369299.369628] Lustre: Skipped 17 previous similar messages [1369899.375467] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1369899.385905] Lustre: Skipped 11 previous similar messages [1370499.392035] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1370499.402475] Lustre: Skipped 12 previous similar messages [1371099.408773] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1371099.419210] Lustre: Skipped 15 previous similar messages [1372000.428268] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1372000.438711] Lustre: Skipped 12 previous similar messages [1372600.445567] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1372600.456011] Lustre: Skipped 15 previous similar messages [1373200.462391] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1373200.472834] Lustre: Skipped 12 previous similar messages [1374099.485227] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1374099.495661] Lustre: Skipped 12 previous similar messages [1374700.498548] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1374700.508977] Lustre: Skipped 11 previous similar messages [1375300.515710] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1375300.526151] Lustre: Skipped 14 previous similar messages [1375900.532646] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1375900.543079] Lustre: Skipped 13 previous similar messages [1376799.552536] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1376799.562992] Lustre: Skipped 12 previous similar messages [1377399.569292] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1377399.579732] Lustre: Skipped 12 previous similar messages [1377999.586216] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1377999.596648] Lustre: Skipped 15 previous similar messages [1378899.608114] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1378899.618553] Lustre: Skipped 12 previous similar messages [1379500.622485] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1379500.632928] Lustre: Skipped 11 previous similar messages [1380399.646853] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1380399.657297] Lustre: Skipped 17 previous similar messages [1380999.663130] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1380999.673564] Lustre: Skipped 13 previous similar messages [1381899.685016] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1381899.695458] Lustre: Skipped 15 previous similar messages [1382500.700920] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1382500.711388] Lustre: Skipped 11 previous similar messages [1383100.717978] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1383100.728419] Lustre: Skipped 16 previous similar messages [1383999.737460] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1383999.747912] Lustre: Skipped 12 previous similar messages [1384600.752821] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1384600.763258] Lustre: Skipped 11 previous similar messages [1385200.768810] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1385200.779284] Lustre: Skipped 13 previous similar messages [1386100.789211] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1386100.799651] Lustre: Skipped 15 previous similar messages [1386999.812808] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1386999.823241] Lustre: Skipped 17 previous similar messages [1387899.833766] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1387899.844204] Lustre: Skipped 17 previous similar messages [1388499.851828] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1388499.862266] Lustre: Skipped 11 previous similar messages [1389399.874769] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1389399.885202] Lustre: Skipped 17 previous similar messages [1390000.889176] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1390000.899606] Lustre: Skipped 11 previous similar messages [1390899.913193] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1390899.923627] Lustre: Skipped 17 previous similar messages [1391800.935203] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1391800.945636] Lustre: Skipped 17 previous similar messages [1392699.956000] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1392699.966443] Lustre: Skipped 17 previous similar messages [1393299.973049] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1393299.983490] Lustre: Skipped 12 previous similar messages [1394199.994159] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1394200.004597] Lustre: Skipped 16 previous similar messages [1394800.011664] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1394800.022098] Lustre: Skipped 15 previous similar messages [1395701.031903] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1395701.042347] Lustre: Skipped 13 previous similar messages [1396600.056304] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1396600.066738] Lustre: Skipped 17 previous similar messages [1397201.070746] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1397201.081186] Lustre: Skipped 11 previous similar messages [1398100.094700] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1398100.105140] Lustre: Skipped 17 previous similar messages [1398700.111530] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1398700.121968] Lustre: Skipped 14 previous similar messages [1399600.132463] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1399600.142898] Lustre: Skipped 14 previous similar messages [1400200.149803] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1400200.160240] Lustre: Skipped 14 previous similar messages [1401100.169689] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1401100.180132] Lustre: Skipped 14 previous similar messages [1401700.186259] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1401700.196699] Lustre: Skipped 11 previous similar messages [1402300.203064] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1402300.213495] Lustre: Skipped 15 previous similar messages [1403200.223758] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1403200.234196] Lustre: Skipped 13 previous similar messages [1403800.242296] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1403800.252729] Lustre: Skipped 12 previous similar messages [1404401.255977] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1404401.266416] Lustre: Skipped 10 previous similar messages [1405301.279091] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1405301.289530] Lustre: Skipped 17 previous similar messages [1406201.303001] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1406201.313441] Lustre: Skipped 17 previous similar messages [1406677.828833] md: data-check of RAID array md5 [1406683.905726] md: data-check of RAID array md11 [1406689.980751] md: data-check of RAID array md9 [1406696.048363] md: data-check of RAID array md3 [1406702.120400] md: data-check of RAID array md1 [1406708.229453] md: data-check of RAID array md7 [1406801.320629] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1406801.331192] Lustre: Skipped 11 previous similar messages [1407700.343311] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1407700.353866] Lustre: Skipped 17 previous similar messages [1408300.359936] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1408300.370383] Lustre: Skipped 13 previous similar messages [1409200.381533] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1409200.391975] Lustre: Skipped 15 previous similar messages [1409800.398146] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1409800.408604] Lustre: Skipped 12 previous similar messages [1410400.415406] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1410400.425842] Lustre: Skipped 15 previous similar messages [1411000.432219] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1411000.442695] Lustre: Skipped 11 previous similar messages [1411900.452273] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1411900.462759] Lustre: Skipped 12 previous similar messages [1412500.469494] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1412500.479987] Lustre: Skipped 12 previous similar messages [1413401.489940] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1413401.500575] Lustre: Skipped 16 previous similar messages [1414301.514532] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1414301.524987] Lustre: Skipped 17 previous similar messages [1415200.538079] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1415200.548514] Lustre: Skipped 17 previous similar messages [1415801.552936] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1415801.563371] Lustre: Skipped 11 previous similar messages [1416700.577981] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1416700.588451] Lustre: Skipped 17 previous similar messages [1417300.593930] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1417300.604478] Lustre: Skipped 12 previous similar messages [1417901.608490] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1417901.618945] Lustre: Skipped 10 previous similar messages [1418800.632691] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1418800.643174] Lustre: Skipped 17 previous similar messages [1419400.649312] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1419400.659750] Lustre: Skipped 12 previous similar messages [1420000.666463] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1420000.676903] Lustre: Skipped 15 previous similar messages [1420900.687760] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1420900.698258] Lustre: Skipped 12 previous similar messages [1421500.705269] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1421500.715730] Lustre: Skipped 12 previous similar messages [1422100.722397] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1422100.732835] Lustre: Skipped 12 previous similar messages [1423000.742034] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1423000.752487] Lustre: Skipped 15 previous similar messages [1423600.758811] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1423600.769272] Lustre: Skipped 12 previous similar messages [1424201.773548] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1424201.784008] Lustre: Skipped 10 previous similar messages [1425100.798303] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1425100.808739] Lustre: Skipped 17 previous similar messages [1425701.813744] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1425701.824183] Lustre: Skipped 11 previous similar messages [1426600.838013] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1426600.848480] Lustre: Skipped 17 previous similar messages [1427200.854568] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1427200.865082] Lustre: Skipped 15 previous similar messages [1427800.871482] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1427800.881924] Lustre: Skipped 10 previous similar messages [1428400.888012] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1428400.898455] Lustre: Skipped 12 previous similar messages [1429300.908697] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1429300.919138] Lustre: Skipped 13 previous similar messages [1429900.926173] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1429900.936619] Lustre: Skipped 12 previous similar messages [1430500.943154] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1430500.953725] Lustre: Skipped 13 previous similar messages [1431400.963782] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1431400.974221] Lustre: Skipped 14 previous similar messages [1432000.981839] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1432000.992282] Lustre: Skipped 15 previous similar messages [1432601.994568] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1432602.005025] Lustre: Skipped 7 previous similar messages [1433501.019186] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1433501.029630] Lustre: Skipped 17 previous similar messages [1434101.037257] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1434101.047701] Lustre: Skipped 11 previous similar messages [1434701.055333] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1434701.065772] Lustre: Skipped 16 previous similar messages [1435602.072340] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1435602.082781] Lustre: Skipped 12 previous similar messages [1436501.097167] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1436501.107685] Lustre: Skipped 17 previous similar messages [1437101.113533] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1437101.123997] Lustre: Skipped 11 previous similar messages [1437701.130578] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1437701.141058] Lustre: Skipped 14 previous similar messages [1438602.150942] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1438602.161383] Lustre: Skipped 14 previous similar messages [1439501.174871] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1439501.185333] Lustre: Skipped 17 previous similar messages [1440102.191671] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1440102.202108] Lustre: Skipped 11 previous similar messages [1441002.213361] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1441002.223814] Lustre: Skipped 17 previous similar messages [1441602.231932] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1441602.242532] Lustre: Skipped 16 previous similar messages [1442202.248423] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1442202.258943] Lustre: Skipped 10 previous similar messages [1443101.272031] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1443101.282483] Lustre: Skipped 13 previous similar messages [1444001.293561] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1444001.304000] Lustre: Skipped 17 previous similar messages [1444601.310751] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1444601.321204] Lustre: Skipped 15 previous similar messages [1445201.328294] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1445201.338767] Lustre: Skipped 12 previous similar messages [1445802.339834] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1445802.350287] Lustre: Skipped 6 previous similar messages [1446701.363807] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1446701.374258] Lustre: Skipped 17 previous similar messages [1447302.379282] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1447302.389760] Lustre: Skipped 11 previous similar messages [1447902.395649] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1447902.406094] Lustre: Skipped 13 previous similar messages [1448502.412354] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1448502.422906] Lustre: Skipped 10 previous similar messages [1449401.435304] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1449401.445748] Lustre: Skipped 16 previous similar messages [1450302.457724] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1450302.468177] Lustre: Skipped 17 previous similar messages [1450902.475772] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1450902.486249] Lustre: Skipped 14 previous similar messages [1451801.496982] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1451801.507437] Lustre: Skipped 14 previous similar messages [1452401.513843] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1452401.524277] Lustre: Skipped 12 previous similar messages [1453001.531478] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1453001.541914] Lustre: Skipped 11 previous similar messages [1453901.552256] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1453901.562693] Lustre: Skipped 16 previous similar messages [1454501.568900] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1454501.579345] Lustre: Skipped 12 previous similar messages [1455101.585509] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1455101.595964] Lustre: Skipped 12 previous similar messages [1456002.607627] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1456002.618073] Lustre: Skipped 15 previous similar messages [1456901.631023] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1456901.641470] Lustre: Skipped 17 previous similar messages [1457501.648452] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1457501.658899] Lustre: Skipped 15 previous similar messages [1458401.668344] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1458401.678793] Lustre: Skipped 13 previous similar messages [1459001.684759] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1459001.695211] Lustre: Skipped 12 previous similar messages [1459601.701202] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1459601.711639] Lustre: Skipped 11 previous similar messages [1460202.715670] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1460202.726135] Lustre: Skipped 10 previous similar messages [1461102.738412] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1461102.748846] Lustre: Skipped 17 previous similar messages [1462001.762028] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1462001.772469] Lustre: Skipped 17 previous similar messages [1462601.778462] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1462601.788900] Lustre: Skipped 11 previous similar messages [1463201.794165] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1463201.804638] Lustre: Skipped 13 previous similar messages [1463801.809989] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1463801.820431] Lustre: Skipped 10 previous similar messages [1464402.823791] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1464402.834236] Lustre: Skipped 10 previous similar messages [1465301.848787] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1465301.859283] Lustre: Skipped 17 previous similar messages [1465901.864864] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1465901.875426] Lustre: Skipped 12 previous similar messages [1466501.881551] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1466501.891992] Lustre: Skipped 14 previous similar messages [1467101.897934] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1467101.908399] Lustre: Skipped 12 previous similar messages [1468001.918415] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1468001.928854] Lustre: Skipped 12 previous similar messages [1468601.934962] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1468601.945405] Lustre: Skipped 12 previous similar messages [1469201.953268] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1469201.963712] Lustre: Skipped 12 previous similar messages [1470102.973165] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1470102.983646] Lustre: Skipped 15 previous similar messages [1471001.997743] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1471002.008176] Lustre: Skipped 17 previous similar messages [1471602.015634] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1471602.026125] Lustre: Skipped 15 previous similar messages [1472203.028392] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1472203.038832] Lustre: Skipped 7 previous similar messages [1473103.052455] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1473103.062911] Lustre: Skipped 17 previous similar messages [1474002.077286] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1474002.087746] Lustre: Skipped 17 previous similar messages [1474603.092222] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1474603.102661] Lustre: Skipped 11 previous similar messages [1475203.108399] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1475203.118873] Lustre: Skipped 12 previous similar messages [1476103.130321] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1476103.140766] Lustre: Skipped 16 previous similar messages [1477003.154161] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1477003.164684] Lustre: Skipped 17 previous similar messages [1477902.180099] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1477902.190627] Lustre: Skipped 17 previous similar messages [1478502.199377] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1478502.209831] Lustre: Skipped 12 previous similar messages [1479403.217697] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1479403.228161] Lustre: Skipped 16 previous similar messages [1480302.245169] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1480302.255634] Lustre: Skipped 17 previous similar messages [1480902.261394] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1480902.271850] Lustre: Skipped 15 previous similar messages [1481802.282008] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1481802.292452] Lustre: Skipped 13 previous similar messages [1482402.301179] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1482402.311624] Lustre: Skipped 13 previous similar messages [1483302.322063] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1483302.332492] Lustre: Skipped 15 previous similar messages [1483903.339053] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1483903.349498] Lustre: Skipped 11 previous similar messages [1484802.362311] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1484802.372753] Lustre: Skipped 17 previous similar messages [1485402.379653] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1485402.390118] Lustre: Skipped 11 previous similar messages [1486002.395673] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1486002.406114] Lustre: Skipped 16 previous similar messages [1486602.412291] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1486602.422723] Lustre: Skipped 9 previous similar messages [1487503.433105] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1487503.443673] Lustre: Skipped 14 previous similar messages [1488402.456903] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1488402.467344] Lustre: Skipped 17 previous similar messages [1489302.481567] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1489302.492016] Lustre: Skipped 17 previous similar messages [1489902.497621] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1489902.508057] Lustre: Skipped 15 previous similar messages [1490502.514503] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1490502.524990] Lustre: Skipped 12 previous similar messages [1491402.535019] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1491402.545465] Lustre: Skipped 12 previous similar messages [1492002.550992] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1492002.561449] Lustre: Skipped 14 previous similar messages [1492903.571094] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1492903.581557] Lustre: Skipped 14 previous similar messages [1493802.595774] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1493802.606223] Lustre: Skipped 17 previous similar messages [1494403.610440] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1494403.620882] Lustre: Skipped 11 previous similar messages [1495302.634322] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1495302.644761] Lustre: Skipped 17 previous similar messages [1495902.652293] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1495902.662724] Lustre: Skipped 11 previous similar messages [1496502.668258] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1496502.678705] Lustre: Skipped 12 previous similar messages [1497103.681888] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1497103.692389] Lustre: Skipped 10 previous similar messages [1497703.699942] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1497703.710400] Lustre: Skipped 14 previous similar messages [1498603.719388] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1498603.729912] Lustre: Skipped 14 previous similar messages [1499502.743440] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1499502.753874] Lustre: Skipped 17 previous similar messages [1500103.757988] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1500103.768522] Lustre: Skipped 11 previous similar messages [1500593.110690] md: md1: data-check done. [1500703.778408] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1500703.788840] Lustre: Skipped 13 previous similar messages [1501602.797668] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1501602.808108] Lustre: Skipped 15 previous similar messages [1502141.560941] md: md11: data-check done. [1502203.813591] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1502203.824036] Lustre: Skipped 11 previous similar messages [1503102.836935] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1503102.847392] Lustre: Skipped 17 previous similar messages [1503702.853912] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1503702.864365] Lustre: Skipped 12 previous similar messages [1504164.861404] md: md3: data-check done. [1504302.870214] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1504302.880652] Lustre: Skipped 14 previous similar messages [1504902.886599] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1504902.897038] Lustre: Skipped 9 previous similar messages [1505802.907163] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1505802.917594] Lustre: Skipped 15 previous similar messages [1506403.921738] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1506403.932179] Lustre: Skipped 11 previous similar messages [1507003.938349] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1507003.948785] Lustre: Skipped 14 previous similar messages [1507902.962341] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1507902.972777] Lustre: Skipped 14 previous similar messages [1508503.976957] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1508503.987413] Lustre: Skipped 11 previous similar messages [1509403.001593] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1509403.012043] Lustre: Skipped 17 previous similar messages [1510004.016120] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1510004.026565] Lustre: Skipped 11 previous similar messages [1510903.040690] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1510903.051126] Lustre: Skipped 17 previous similar messages [1511503.058326] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1511503.068771] Lustre: Skipped 12 previous similar messages [1512103.075769] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1512103.086206] Lustre: Skipped 13 previous similar messages [1513003.094639] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1513003.105091] Lustre: Skipped 14 previous similar messages [1513604.110541] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1513604.121001] Lustre: Skipped 11 previous similar messages [1514503.135088] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1514503.145525] Lustre: Skipped 17 previous similar messages [1515103.151589] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1515103.162029] Lustre: Skipped 12 previous similar messages [1516003.156524] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1516003.166960] Lustre: Skipped 16 previous similar messages [1516603.174013] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1516603.184446] Lustre: Skipped 11 previous similar messages [1517203.190124] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1517203.200564] Lustre: Skipped 12 previous similar messages [1517803.205546] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1517803.216030] Lustre: Skipped 11 previous similar messages [1518403.235889] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1518403.246330] Lustre: Skipped 10 previous similar messages [1519003.253699] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1519003.264142] Lustre: Skipped 18 previous similar messages [1519603.272101] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1519603.282540] Lustre: Skipped 14 previous similar messages [1520204.286195] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1520204.296655] Lustre: Skipped 7 previous similar messages [1521103.308906] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1521103.319350] Lustre: Skipped 17 previous similar messages [1521703.326130] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1521703.336581] Lustre: Skipped 16 previous similar messages [1522604.345574] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1522604.356009] Lustre: Skipped 12 previous similar messages [1523504.369641] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1523504.380079] Lustre: Skipped 17 previous similar messages [1524256.956898] Lustre: fir-OST000b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1524256.967246] Lustre: Skipped 17 previous similar messages [1525003.407109] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1525003.417546] Lustre: Skipped 18 previous similar messages [1525688.438552] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1525688.448993] Lustre: Skipped 23 previous similar messages [1526503.446660] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1526503.457100] Lustre: Skipped 17 previous similar messages [1527103.463623] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1527103.474097] Lustre: Skipped 17 previous similar messages [1527703.479581] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1527703.490030] Lustre: Skipped 12 previous similar messages [1528303.496520] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1528303.506974] Lustre: Skipped 12 previous similar messages [1528904.511327] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1528904.521760] Lustre: Skipped 9 previous similar messages [1529504.528485] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1529504.538922] Lustre: Skipped 15 previous similar messages [1530403.550784] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1530403.561225] Lustre: Skipped 13 previous similar messages [1531003.567321] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1531003.577761] Lustre: Skipped 13 previous similar messages [1531604.580137] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1531604.590571] Lustre: Skipped 9 previous similar messages [1532504.606707] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1532504.617198] Lustre: Skipped 17 previous similar messages [1533404.627639] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1533404.638089] Lustre: Skipped 17 previous similar messages [1534303.651618] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1534303.662085] Lustre: Skipped 17 previous similar messages [1534903.668023] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1534903.678469] Lustre: Skipped 13 previous similar messages [1535503.684623] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1535503.695070] Lustre: Skipped 12 previous similar messages [1536103.699761] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1536103.710214] Lustre: Skipped 11 previous similar messages [1536703.715495] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1536703.725947] Lustre: Skipped 11 previous similar messages [1537603.736151] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1537603.746637] Lustre: Skipped 14 previous similar messages [1538203.752373] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1538203.762853] Lustre: Skipped 11 previous similar messages [1538804.767731] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1538804.778181] Lustre: Skipped 11 previous similar messages [1539404.784967] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1539404.795463] Lustre: Skipped 18 previous similar messages [1540004.801049] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1540004.811550] Lustre: Skipped 15 previous similar messages [1540903.824206] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1540903.834644] Lustre: Skipped 31 previous similar messages [1541504.838539] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1541504.848983] Lustre: Skipped 33 previous similar messages [1542104.854843] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1542104.865298] Lustre: Skipped 17 previous similar messages [1542726.273844] Lustre: fir-OST0001: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1542726.284275] Lustre: Skipped 55 previous similar messages [1543603.876340] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1543603.886821] Lustre: Skipped 29 previous similar messages [1544203.893204] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1544203.903692] Lustre: Skipped 23 previous similar messages [1544803.908928] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1544803.919369] Lustre: Skipped 29 previous similar messages [1545403.924985] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1545403.935461] Lustre: Skipped 51 previous similar messages [1546079.187073] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1546079.197565] Lustre: Skipped 60 previous similar messages [1546679.789554] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1546679.800044] Lustre: Skipped 25 previous similar messages [1547298.486989] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1547298.497448] Lustre: Skipped 31 previous similar messages [1547916.327138] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1547916.337590] Lustre: Skipped 43 previous similar messages [1548533.534451] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1548533.544896] Lustre: Skipped 44 previous similar messages [1549139.351427] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1549139.361871] Lustre: Skipped 45 previous similar messages [1549904.041543] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1549904.051980] Lustre: Skipped 15 previous similar messages [1550504.059464] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1550504.069916] Lustre: Skipped 11 previous similar messages [1551104.076218] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1551104.086682] Lustre: Skipped 17 previous similar messages [1551704.091260] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1551704.101716] Lustre: Skipped 20 previous similar messages [1552304.106671] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1552304.117106] Lustre: Skipped 12 previous similar messages [1552904.122183] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1552904.132622] Lustre: Skipped 17 previous similar messages [1553504.139294] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1553504.149736] Lustre: Skipped 15 previous similar messages [1554104.154406] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1554104.164862] Lustre: Skipped 44 previous similar messages [1554704.186577] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1554704.197013] Lustre: Skipped 22 previous similar messages [1555304.202995] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1555304.213440] Lustre: Skipped 11 previous similar messages [1555904.219688] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1555904.230126] Lustre: Skipped 14 previous similar messages [1556505.234032] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1556505.244495] Lustre: Skipped 14 previous similar messages [1557405.257769] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1557405.268206] Lustre: Skipped 17 previous similar messages [1558005.274594] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1558005.285093] Lustre: Skipped 17 previous similar messages [1558905.298563] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1558905.309015] Lustre: Skipped 17 previous similar messages [1559568.548101] Lustre: fir-OST0001: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1559568.558556] Lustre: Skipped 27 previous similar messages [1560036.529233] md: md5: data-check done. [1560073.227173] md: md7: data-check done. [1560404.337447] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1560404.347885] Lustre: Skipped 13 previous similar messages [1561004.354286] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1561004.364722] Lustre: Skipped 14 previous similar messages [1561904.377844] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1561904.388303] Lustre: Skipped 22 previous similar messages [1562504.394381] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1562504.404817] Lustre: Skipped 14 previous similar messages [1562818.654286] md: md9: data-check done. [1563218.089356] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1563218.099791] Lustre: Skipped 31 previous similar messages [1563848.965529] Lustre: fir-OST0001: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1563848.975964] Lustre: Skipped 30 previous similar messages [1564604.446370] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1564604.456810] Lustre: Skipped 17 previous similar messages [1565204.463166] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1565204.473595] Lustre: Skipped 12 previous similar messages [1565804.479173] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1565804.489617] Lustre: Skipped 23 previous similar messages [1566404.496835] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1566404.507278] Lustre: Skipped 22 previous similar messages [1567281.187709] Lustre: fir-OST0003: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1567281.198139] Lustre: Skipped 19 previous similar messages [1567904.533624] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1567904.544056] Lustre: Skipped 12 previous similar messages [1568505.551362] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1568505.561796] Lustre: Skipped 21 previous similar messages [1569213.310588] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1569213.321030] Lustre: Skipped 27 previous similar messages [1570004.590813] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1570004.601250] Lustre: Skipped 28 previous similar messages [1570646.803141] Lustre: fir-OST0003: Connection restored to cc9aeebd-5eb1-fd0a-28e5-4a7c51fd2782 (at 10.8.6.33@o2ib6) [1570646.813588] Lustre: Skipped 26 previous similar messages [1571504.614808] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1571504.625239] Lustre: Skipped 28 previous similar messages [1572105.646090] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1572105.656534] Lustre: Skipped 14 previous similar messages [1572705.664079] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1572705.674521] Lustre: Skipped 31 previous similar messages [1573452.942356] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1573452.952791] Lustre: Skipped 14 previous similar messages [1574150.757136] Lustre: fir-OST0001: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1574150.767576] Lustre: Skipped 17 previous similar messages [1574804.700182] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1574804.710628] Lustre: Skipped 12 previous similar messages [1575404.716515] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1575404.726955] Lustre: Skipped 17 previous similar messages [1576004.732167] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1576004.742607] Lustre: Skipped 17 previous similar messages [1576604.748650] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1576604.759088] Lustre: Skipped 12 previous similar messages [1577204.764182] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1577204.774621] Lustre: Skipped 11 previous similar messages [1577804.781012] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1577804.791448] Lustre: Skipped 16 previous similar messages [1578404.797020] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1578404.807454] Lustre: Skipped 12 previous similar messages [1579004.812199] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1579004.822631] Lustre: Skipped 13 previous similar messages [1579604.827527] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1579604.837968] Lustre: Skipped 18 previous similar messages [1580204.843564] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1580204.853997] Lustre: Skipped 10 previous similar messages [1580804.875731] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1580804.886174] Lustre: Skipped 11 previous similar messages [1581404.891752] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1581404.902187] Lustre: Skipped 17 previous similar messages [1582004.908834] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1582004.919285] Lustre: Skipped 12 previous similar messages [1582604.925611] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1582604.936046] Lustre: Skipped 12 previous similar messages [1583205.939513] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1583205.949946] Lustre: Skipped 14 previous similar messages [1584105.961492] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1584105.971930] Lustre: Skipped 17 previous similar messages [1585005.985155] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1585005.995591] Lustre: Skipped 23 previous similar messages [1585606.002524] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1585606.012961] Lustre: Skipped 12 previous similar messages [1586506.024793] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1586506.035230] Lustre: Skipped 22 previous similar messages [1587405.048455] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1587405.058890] Lustre: Skipped 17 previous similar messages [1588006.063379] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1588006.073825] Lustre: Skipped 17 previous similar messages [1588905.088201] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1588905.098632] Lustre: Skipped 17 previous similar messages [1589506.103793] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1589506.114231] Lustre: Skipped 11 previous similar messages [1590406.126261] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1590406.136705] Lustre: Skipped 23 previous similar messages [1591238.662221] Lustre: fir-OST0001: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1591238.672666] Lustre: Skipped 22 previous similar messages [1591905.168298] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1591905.178733] Lustre: Skipped 12 previous similar messages [1592506.182484] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1592506.192924] Lustre: Skipped 11 previous similar messages [1593106.198655] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1593106.209088] Lustre: Skipped 18 previous similar messages [1594005.220534] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1594005.230973] Lustre: Skipped 15 previous similar messages [1594606.237233] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1594606.247666] Lustre: Skipped 11 previous similar messages [1595206.253641] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1595206.264072] Lustre: Skipped 19 previous similar messages [1596105.277674] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1596105.288121] Lustre: Skipped 15 previous similar messages [1596706.293419] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1596706.303857] Lustre: Skipped 17 previous similar messages [1597605.317875] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1597605.328306] Lustre: Skipped 17 previous similar messages [1598205.334467] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1598205.344906] Lustre: Skipped 15 previous similar messages [1599105.356269] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1599105.366704] Lustre: Skipped 19 previous similar messages [1599706.372498] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1599706.382937] Lustre: Skipped 17 previous similar messages [1600605.396626] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1600605.407060] Lustre: Skipped 17 previous similar messages [1601205.411993] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1601205.422427] Lustre: Skipped 20 previous similar messages [1601805.427591] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1601805.438038] Lustre: Skipped 15 previous similar messages [1602405.444282] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1602405.454718] Lustre: Skipped 13 previous similar messages [1603005.461107] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1603005.471556] Lustre: Skipped 13 previous similar messages [1603606.473958] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1603606.484396] Lustre: Skipped 12 previous similar messages [1604506.498216] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1604506.508647] Lustre: Skipped 17 previous similar messages [1605106.514575] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1605106.525011] Lustre: Skipped 23 previous similar messages [1606005.540507] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1606005.550945] Lustre: Skipped 18 previous similar messages [1606605.556549] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1606605.566987] Lustre: Skipped 19 previous similar messages [1607505.577904] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1607505.588347] Lustre: Skipped 15 previous similar messages [1608105.594375] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1608105.604812] Lustre: Skipped 18 previous similar messages [1608705.610714] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1608705.621146] Lustre: Skipped 9 previous similar messages [1609305.626990] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1609305.637422] Lustre: Skipped 14 previous similar messages [1609905.643160] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1609905.653589] Lustre: Skipped 17 previous similar messages [1610506.656566] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1610506.667005] Lustre: Skipped 8 previous similar messages [1611106.673393] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1611106.683825] Lustre: Skipped 37 previous similar messages [1611722.286432] Lustre: fir-OST0009: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1611722.297049] Lustre: Skipped 21 previous similar messages [1612335.187408] Lustre: fir-OST0009: Connection restored to 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) [1612335.198018] Lustre: Skipped 17 previous similar messages [1613206.727868] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1613206.738311] Lustre: Skipped 37 previous similar messages [1613806.743571] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1613806.754001] Lustre: Skipped 11 previous similar messages [1614620.522799] Lustre: fir-OST0001: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1614620.533423] Lustre: Skipped 46 previous similar messages [1615301.457606] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1615301.468048] Lustre: Skipped 17 previous similar messages [1615905.798902] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1615905.809337] Lustre: Skipped 12 previous similar messages [1616506.814265] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1616506.824698] Lustre: Skipped 12 previous similar messages [1617291.369323] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1617291.379767] Lustre: Skipped 61 previous similar messages [1617835.914539] Lustre: fir-OST000b: haven't heard from client f844fdc2-4b3f-a5e3-d40e-580f6f5c3bcb (at 10.9.101.59@o2ib4) in 228 seconds. I think it's dead, and I am evicting it. exp ffff90cbfbed8000, cur 1549395554 expire 1549395404 last 1549395326 [1617835.936511] Lustre: Skipped 5 previous similar messages [1617838.252635] Lustre: fir-OST0009: haven't heard from client f844fdc2-4b3f-a5e3-d40e-580f6f5c3bcb (at 10.9.101.59@o2ib4) in 230 seconds. I think it's dead, and I am evicting it. exp ffff90cbfbedb800, cur 1549395556 expire 1549395406 last 1549395326 [1617896.267375] Lustre: fir-OST0001: Connection restored to 2708a86e-3625-5509-9d0d-aaed4fc61a05 (at 10.8.1.31@o2ib6) [1617896.277827] Lustre: Skipped 33 previous similar messages [1618606.869898] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1618606.880336] Lustre: Skipped 36 previous similar messages [1619218.081501] Lustre: fir-OST0001: Connection restored to 3c005caf-e304-dd1a-234a-65ceb646a930 (at 10.8.1.2@o2ib6) [1619218.081502] Lustre: fir-OST0005: Connection restored to 3c005caf-e304-dd1a-234a-65ceb646a930 (at 10.8.1.2@o2ib6) [1619218.081505] Lustre: Skipped 17 previous similar messages [1619218.107679] Lustre: Skipped 4 previous similar messages [1619842.919101] Lustre: fir-OST0003: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1619842.919102] Lustre: fir-OST0001: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1619842.919103] Lustre: fir-OST0005: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1619842.919104] Lustre: Skipped 18 previous similar messages [1619842.919107] Lustre: Skipped 18 previous similar messages [1619842.961417] Lustre: Skipped 3 previous similar messages [1620750.979566] Lustre: fir-OST0007: haven't heard from client 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90c91ec9b000, cur 1549398469 expire 1549398319 last 1549398242 [1621377.102687] Lustre: fir-OST0001: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1621377.113123] Lustre: Skipped 101 previous similar messages [1621567.254519] Lustre: fir-OST0001: Connection restored to 775ad82c-450c-c16e-bb21-700cda6d4afd (at 10.9.103.39@o2ib4) [1621567.265141] Lustre: Skipped 17 previous similar messages [1622940.999575] Lustre: fir-OST0001: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1622941.010013] Lustre: Skipped 17 previous similar messages [1622963.326315] Lustre: fir-OST0001: Connection restored to 8ae4cc68-204d-d2bb-4908-94f49692744b (at 10.8.20.15@o2ib6) [1622963.336838] Lustre: Skipped 11 previous similar messages [1623795.080666] Lustre: fir-OST0001: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [1623795.091027] Lustre: Skipped 5 previous similar messages [1624021.746937] LNetError: 79895:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds [1624021.757280] LNetError: 79895:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Skipped 2 previous similar messages [1624021.767620] LNetError: 79895:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.201@o2ib7 (49): c: 6, oc: 0, rc: 8 [1624021.779864] LNetError: 79895:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Skipped 2 previous similar messages [1624184.807548] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1624188.105689] Lustre: fir-OST0003: haven't heard from client e288d2fa-1494-4b6c-352f-7c16658a712b (at 10.8.1.1@o2ib6) in 229 seconds. I think it's dead, and I am evicting it. exp ffff90b2c9564000, cur 1549401906 expire 1549401756 last 1549401677 [1624188.127405] Lustre: Skipped 5 previous similar messages [1624316.754494] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 296 seconds [1624316.765008] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 8 previous similar messages [1624319.754599] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.209@o2ib7: 1 seconds [1624319.764944] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1624617.762195] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 597 seconds [1624617.772709] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [1624918.769904] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 898 seconds [1624918.780420] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [1625219.777607] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 1199 seconds [1625219.788213] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [1626123.800797] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 2103 seconds [1626128.800887] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.201@o2ib7: 1 seconds [1626128.811226] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 4 previous similar messages [1626468.291266] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1626468.301637] Lustre: Skipped 5 previous similar messages [1626469.704245] Lustre: fir-OST0007: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1626477.203505] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1626482.893892] Lustre: fir-OST0003: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1627801.339124] Lustre: fir-OST0001: Connection restored to 6da5269e-e6e7-e930-ea8b-e990b1fd18b0 (at 10.9.101.72@o2ib4) [1627801.349739] Lustre: Skipped 7 previous similar messages [1627802.608281] Lustre: fir-OST0001: Connection restored to fd4739f5-61ac-b1c8-a4a0-9ad8b819daba (at 10.9.101.38@o2ib4) [1627802.618913] Lustre: Skipped 29 previous similar messages [1627805.018674] Lustre: fir-OST0001: Connection restored to 7ac937e4-43b1-06ba-e012-59210bc9ec60 (at 10.9.101.18@o2ib4) [1627805.029294] Lustre: Skipped 101 previous similar messages [1627859.493653] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1627859.504085] Lustre: Skipped 36 previous similar messages [1627898.949834] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1627898.960281] Lustre: Skipped 1 previous similar message [1627985.814671] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1627985.825113] Lustre: Skipped 3 previous similar messages [1628206.119830] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1628506.129772] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1628506.140205] Lustre: Skipped 5 previous similar messages [1628806.137809] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1628806.148248] Lustre: Skipped 5 previous similar messages [1629106.146673] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1629106.157118] Lustre: Skipped 5 previous similar messages [1629706.161403] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1629706.171842] Lustre: Skipped 75 previous similar messages [1630306.184442] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1630306.194876] Lustre: Skipped 378 previous similar messages [1630912.381420] Lustre: fir-OST0001: Connection restored to f836dc38-744c-c596-c0a2-f4034f572070 (at 10.9.102.15@o2ib4) [1630912.392033] Lustre: Skipped 104 previous similar messages [1631528.631046] Lustre: fir-OST0001: Connection restored to 75d3df1c-c05f-80f2-4e0d-3991a06b4cf2 (at 10.8.2.9@o2ib6) [1631528.641429] Lustre: Skipped 611 previous similar messages [1632131.603106] Lustre: fir-OST0001: Connection restored to 3ce6d475-6724-c117-8c44-da8378e50030 (at 10.9.101.69@o2ib4) [1632131.613718] Lustre: Skipped 353 previous similar messages [1632784.428126] Lustre: fir-OST0001: Connection restored to e033b2fb-58ee-ad20-dbe1-c069873ac977 (at 10.9.101.47@o2ib4) [1632784.438738] Lustre: Skipped 53 previous similar messages [1633434.001227] Lustre: fir-OST0001: Connection restored to ecc5b628-5efd-fbfd-7392-a1abe17de407 (at 10.9.101.36@o2ib4) [1633434.011836] Lustre: Skipped 32 previous similar messages [1634047.400900] Lustre: fir-OST000b: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [1634047.411253] Lustre: Skipped 61 previous similar messages [1634649.640689] Lustre: fir-OST0003: Connection restored to dba0ac03-f756-7e65-44de-ab2d15018d1d (at 10.9.101.41@o2ib4) [1634649.651304] Lustre: Skipped 69 previous similar messages [1635274.839241] Lustre: fir-OST0003: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [1635274.849585] Lustre: Skipped 70 previous similar messages [1635992.565179] Lustre: fir-OST000b: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [1635992.575526] Lustre: Skipped 62 previous similar messages [1637060.155185] Lustre: fir-OST0003: Connection restored to 61d0fcbf-b5c3-a78f-9cdf-55e61d025648 (at 10.9.0.81@o2ib4) [1637060.165620] Lustre: Skipped 24 previous similar messages [1638433.455235] Lustre: fir-OST0005: haven't heard from client 673a177f-1230-a616-2a0b-8995b4882735 (at 10.9.0.1@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90c7d63da000, cur 1549416151 expire 1549416001 last 1549415924 [1638433.476943] Lustre: Skipped 44 previous similar messages [1638434.507165] Lustre: fir-OST000b: haven't heard from client 673a177f-1230-a616-2a0b-8995b4882735 (at 10.9.0.1@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90abf9fbf400, cur 1549416152 expire 1549416002 last 1549415925 [1638434.528872] Lustre: Skipped 2 previous similar messages [1638481.752933] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1638481.763304] Lustre: Skipped 8 previous similar messages [1638708.583503] Lustre: fir-OST000b: haven't heard from client 673a177f-1230-a616-2a0b-8995b4882735 (at 10.9.0.1@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90bebb89d800, cur 1549416426 expire 1549416276 last 1549416199 [1638708.605221] Lustre: Skipped 1 previous similar message [1638753.524497] Lustre: fir-OST0009: haven't heard from client 673a177f-1230-a616-2a0b-8995b4882735 (at 10.9.0.1@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90b29edaec00, cur 1549416471 expire 1549416321 last 1549416244 [1638753.546198] Lustre: Skipped 1 previous similar message [1638898.048991] Lustre: fir-OST0001: Connection restored to 948ddf77-30e8-6f7f-29c3-d9d6fe8d8435 (at 10.9.101.4@o2ib4) [1638898.059526] Lustre: Skipped 3 previous similar messages [1639081.272432] Lustre: fir-OST0003: Connection restored to 948ddf77-30e8-6f7f-29c3-d9d6fe8d8435 (at 10.9.101.4@o2ib4) [1639081.282951] Lustre: Skipped 26 previous similar messages [1639429.085019] Lustre: fir-OST0001: Connection restored to c46b7f3d-ec55-7fd7-e207-e9c6f5525b60 (at 10.8.0.68@o2ib6) [1639429.095451] Lustre: Skipped 27 previous similar messages [1639560.350661] Lustre: 103469:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417270/real 1549417270] req@ffff90c2224de600 x1623662750915936/t0(0) o104->fir-OST0009@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417277 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1639560.378000] Lustre: 103469:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 35 previous similar messages [1639565.473789] Lustre: 130579:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417276/real 1549417276] req@ffff90c2224dbc00 x1623662750920704/t0(0) o104->fir-OST0003@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417283 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1639565.501173] Lustre: 130579:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1639567.351834] Lustre: 5136:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417277/real 1549417277] req@ffff90c48445c200 x1623662750916016/t0(0) o104->fir-OST0009@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417284 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639567.379051] Lustre: 5136:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 40 previous similar messages [1639572.475973] Lustre: 56693:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417283/real 1549417283] req@ffff90c484476600 x1623662750921504/t0(0) o104->fir-OST0005@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417290 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639579.503149] Lustre: 56693:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417290/real 1549417290] req@ffff90c484476600 x1623662750921504/t0(0) o104->fir-OST0005@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417297 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639579.530403] Lustre: 56693:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 51 previous similar messages [1639588.352382] Lustre: 56688:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417298/real 1549417298] req@ffff90be2ec36c00 x1623662750916128/t0(0) o104->fir-OST0009@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417305 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639588.352384] Lustre: 65710:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417298/real 1549417298] req@ffff90c484473300 x1623662750916224/t0(0) o104->fir-OST0009@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417305 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639588.352388] Lustre: 65710:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 44 previous similar messages [1639588.416891] Lustre: 56688:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 37 previous similar messages [1639607.511880] Lustre: 130579:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417318/real 1549417318] req@ffff90c2224dbc00 x1623662750920704/t0(0) o104->fir-OST0003@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417325 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639607.539252] Lustre: 130579:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 100 previous similar messages [1639642.517763] Lustre: 103235:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417353/real 1549417353] req@ffff90c484459500 x1623662750921936/t0(0) o104->fir-OST0001@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417360 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639642.545104] Lustre: 103235:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 226 previous similar messages [1639658.354180] LustreError: 65710:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) ### client (nid 10.9.0.2@o2ib4) failed to reply to blocking AST (req@ffff90c484473300 x1623662750916224 status 0 rc -110), evict it ns: filter-fir-OST0009_UUID lock: ffff90cb2eb05e80/0xda500c83d2dd4565 lrc: 4/0,0 mode: PR/PR res: [0x30ef8:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c2a0a25 expref: 35 pid: 56756 timeout: 1639708 lvb_type: 1 [1639658.355180] LustreError: 138-a: fir-OST0009: A client on nid 10.9.0.2@o2ib4 was evicted due to a lock blocking callback time out: rc -110 [1639658.355207] LustreError: 81679:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 105s: evicting client at 10.9.0.2@o2ib4 ns: filter-fir-OST0009_UUID lock: ffff90c179ab4ec0/0xda500c83d2dd46fb lrc: 3/0,0 mode: PR/PR res: [0x30de5:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c2a1404 expref: 36 pid: 87621 timeout: 0 lvb_type: 1 [1639658.362566] LustreError: 103474:0:(client.c:1175:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@ffff90cb4e029500 x1623662750976832/t0(0) o104->fir-OST0005@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 0 dl 0 ref 1 fl Rpc:/0/ffffffff rc 0/-1 [1639658.472922] LustreError: 65710:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) Skipped 8 previous similar messages [1639707.365420] Lustre: 5028:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417417/real 1549417417] req@ffff90c2224dd100 x1623662750918432/t0(0) o104->fir-OST0001@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417424 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639707.365448] LustreError: 50359:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) ### client (nid 10.9.0.2@o2ib4) failed to reply to blocking AST (req@ffff90c484472d00 x1623662750918624 status 0 rc -110), evict it ns: filter-fir-OST0001_UUID lock: ffff90c746a50b40/0xda500c83d2dd3c82 lrc: 4/0,0 mode: PR/PR res: [0x2a97a:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c29be3d expref: 33 pid: 82413 timeout: 1639807 lvb_type: 1 [1639707.365453] LustreError: 138-a: fir-OST0001: A client on nid 10.9.0.2@o2ib4 was evicted due to a lock blocking callback time out: rc -110 [1639707.365455] LustreError: Skipped 9 previous similar messages [1639707.365478] LustreError: 81679:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 154s: evicting client at 10.9.0.2@o2ib4 ns: filter-fir-OST0001_UUID lock: ffff90c746a50b40/0xda500c83d2dd3c82 lrc: 3/0,0 mode: PR/PR res: [0x2a97a:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c29be3d expref: 34 pid: 82413 timeout: 0 lvb_type: 1 [1639707.365480] LustreError: 81679:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 5 previous similar messages [1639707.505951] Lustre: 5028:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 165 previous similar messages [1640067.224223] Lustre: fir-OST0001: Connection restored to 948ddf77-30e8-6f7f-29c3-d9d6fe8d8435 (at 10.9.101.4@o2ib4) [1640067.234748] Lustre: Skipped 74 previous similar messages [1640669.450178] Lustre: fir-OST0009: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1640669.460703] Lustre: Skipped 43 previous similar messages [1641624.699298] Lustre: fir-OST0001: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1641624.709820] Lustre: Skipped 42 previous similar messages [1642248.213661] Lustre: fir-OST0009: Connection restored to c46b7f3d-ec55-7fd7-e207-e9c6f5525b60 (at 10.8.0.68@o2ib6) [1642248.224094] Lustre: Skipped 24 previous similar messages [1642997.325114] Lustre: fir-OST0003: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1642997.335641] Lustre: Skipped 25 previous similar messages [1643751.906704] Lustre: fir-OST0001: Connection restored to 0732bf9f-f5f9-3f7f-d05c-6eafb128d46c (at 10.8.0.66@o2ib6) [1643751.917149] Lustre: Skipped 43 previous similar messages [1644380.009810] Lustre: fir-OST0001: Connection restored to dfbc3ca1-e3f2-2f8c-ef1d-6d31c339e5cc (at 10.9.103.31@o2ib4) [1644380.020425] Lustre: Skipped 10 previous similar messages [1645020.457483] Lustre: fir-OST0001: Connection restored to c46b7f3d-ec55-7fd7-e207-e9c6f5525b60 (at 10.8.0.68@o2ib6) [1645020.467918] Lustre: Skipped 32 previous similar messages [1645701.727593] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1645701.738031] Lustre: Skipped 25 previous similar messages [1646313.927053] Lustre: fir-OST0001: Connection restored to c46b7f3d-ec55-7fd7-e207-e9c6f5525b60 (at 10.8.0.68@o2ib6) [1646313.937490] Lustre: Skipped 35 previous similar messages [1646935.938806] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1646935.949241] Lustre: Skipped 41 previous similar messages [1647615.883201] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1647615.893709] Lustre: Skipped 39 previous similar messages [1648236.704129] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1648236.714568] Lustre: Skipped 27 previous similar messages [1648840.336714] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1648840.347146] Lustre: Skipped 48 previous similar messages [1649775.036817] Lustre: fir-OST0001: Connection restored to 0732bf9f-f5f9-3f7f-d05c-6eafb128d46c (at 10.8.0.66@o2ib6) [1649775.047249] Lustre: Skipped 17 previous similar messages [1650595.188570] Lustre: fir-OST0001: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1650595.198922] Lustre: Skipped 27 previous similar messages [1651676.181620] Lustre: fir-OST0003: Connection restored to 55a5f43f-2070-1278-9f4b-9a1845c8940d (at 10.8.11.22@o2ib6) [1651676.192142] Lustre: Skipped 4 previous similar messages [1653200.489818] Lustre: fir-OST0001: Connection restored to 55a5f43f-2070-1278-9f4b-9a1845c8940d (at 10.8.11.22@o2ib6) [1653200.500338] Lustre: Skipped 10 previous similar messages [1653947.283276] Lustre: fir-OST0001: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1653947.283277] Lustre: fir-OST0003: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1653947.304144] Lustre: Skipped 4 previous similar messages [1654226.924933] Lustre: fir-OST0007: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1654230.175349] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1654230.185790] Lustre: Skipped 7 previous similar messages [1654270.857618] Lustre: fir-OST0005: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1654270.867964] Lustre: Skipped 2 previous similar messages [1654290.865445] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1654290.875901] Lustre: Skipped 5 previous similar messages [1654346.967631] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1654346.978065] Lustre: Skipped 3 previous similar messages [1654384.613574] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1654384.624016] Lustre: Skipped 5 previous similar messages [1654529.897414] Lustre: fir-OST0003: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1654529.907854] Lustre: Skipped 2 previous similar messages [1654680.449322] Lustre: fir-OST0001: Connection restored to b34755b7-d181-850b-8754-b1f884364504 (at 10.9.103.44@o2ib4) [1654680.459938] Lustre: Skipped 46 previous similar messages [1654996.745703] Lustre: fir-OST0001: Connection restored to 02a649b1-8bdf-b8d1-3375-fceaf78b0f08 (at 10.9.103.1@o2ib4) [1654996.756228] Lustre: Skipped 99 previous similar messages [1655703.412902] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1655703.423338] Lustre: Skipped 124 previous similar messages [1656307.154594] Lustre: fir-OST0001: Connection restored to afd89a08-6b67-04f7-2303-d504e382d3cd (at 10.9.105.61@o2ib4) [1656307.165237] Lustre: Skipped 521 previous similar messages [1656951.817972] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1656951.828410] Lustre: Skipped 517 previous similar messages [1657552.167544] Lustre: fir-OST0001: Connection restored to 48d1a8fb-a4f7-b303-3034-25b4451574ef (at 10.9.106.37@o2ib4) [1657552.178154] Lustre: Skipped 182 previous similar messages [1658198.213936] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1658198.224380] Lustre: Skipped 102 previous similar messages [1658805.579492] Lustre: fir-OST0001: Connection restored to 10f5ca51-c1e4-ea2c-70bb-5e5b8eb0ed33 (at 10.9.115.7@o2ib4) [1658805.590024] Lustre: Skipped 112 previous similar messages [1659406.942696] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1659406.953132] Lustre: Skipped 162 previous similar messages [1660007.957611] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1660007.968045] Lustre: Skipped 13 previous similar messages [1660607.975859] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1660607.986290] Lustre: Skipped 24 previous similar messages [1661272.106082] Lustre: fir-OST0001: Connection restored to 8ae4cc68-204d-d2bb-4908-94f49692744b (at 10.8.20.15@o2ib6) [1661272.116609] Lustre: Skipped 19 previous similar messages [1661873.489938] Lustre: fir-OST0001: Connection restored to 0e25277b-1ad0-ec5f-2777-56d7cafdcd31 (at 10.8.17.6@o2ib6) [1661873.500377] Lustre: Skipped 343 previous similar messages [1662531.724016] Lustre: fir-OST0001: Connection restored to e4e1992b-645c-7fa0-95fb-f97f854d138b (at 10.8.26.23@o2ib6) [1662531.734538] Lustre: Skipped 723 previous similar messages [1663199.784491] Lustre: fir-OST0005: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1663199.794924] Lustre: Skipped 14 previous similar messages [1663802.916932] Lustre: fir-OST0001: Connection restored to ed8a9d5a-3c64-88a2-4de9-5c7913d6ef08 (at 10.9.101.15@o2ib4) [1663802.927547] Lustre: Skipped 888 previous similar messages [1664439.025150] Lustre: fir-OST0003: Connection restored to e9d1b5f8-7ec9-998b-fe00-a6102cb74525 (at 10.9.102.2@o2ib4) [1664439.035673] Lustre: Skipped 332 previous similar messages [1665107.090794] Lustre: fir-OST0001: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1665107.101235] Lustre: Skipped 133 previous similar messages [1665273.030852] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665273.041648] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549442989/real 1549442989] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443000 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [1665273.069420] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7277 previous similar messages [1665273.531778] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665273.542561] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 113347 previous similar messages [1665274.531804] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665274.542583] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 206460 previous similar messages [1665276.531857] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665276.542636] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 411717 previous similar messages [1665280.531958] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665280.542732] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 818226 previous similar messages [1665288.532163] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665288.542948] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 1636387 previous similar messages [1665289.042178] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443005/real 1549443005] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443016 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665289.069946] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3284985 previous similar messages [1665304.532598] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665304.543381] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 3288438 previous similar messages [1665311.763933] Lustre: 82414:0:(service.c:2165:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (6:1s); client may timeout. req@ffff90dbddb8fc50 x1624673485499552/t0(0) o9->1753156e-3e53-ffdc-53f8-4f5717fb63ea@10.9.102.27@o2ib4:317/0 lens 224/192 e 0 to 0 dl 1549443027 ref 1 fl Complete:/0/0 rc 0/0 [1665311.792230] Lustre: 82414:0:(service.c:2165:ptlrpc_server_handle_request()) Skipped 2 previous similar messages [1665321.042996] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443037/real 1549443037] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443048 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665321.070765] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6590726 previous similar messages [1665323.756023] Lustre: 84382:0:(service.c:2165:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (6:2s); client may timeout. req@ffff90bbee255450 x1624705153056176/t0(0) o9->3d0b5fd2-0a59-4302-6a63-f9505f28fab2@10.8.10.18@o2ib6:328/0 lens 224/192 e 0 to 0 dl 1549443038 ref 1 fl Complete:/0/0 rc 0/0 [1665323.784228] Lustre: 84382:0:(service.c:2165:ptlrpc_server_handle_request()) Skipped 1 previous similar message [1665325.711186] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.8.8.22@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665336.533400] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665336.544188] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 6605915 previous similar messages [1665369.017093] Lustre: fir-OST0003: Client 82c763a5-8dc0-7fc4-d90b-e4497b03f725 (at 10.9.104.57@o2ib4) reconnecting [1665369.027439] Lustre: Skipped 6 previous similar messages [1665385.044637] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443101/real 1549443101] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443112 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665385.072417] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 13258521 previous similar messages [1665400.535029] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665400.545808] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 13233837 previous similar messages [1665473.449909] LNet: Service thread pid 85918 was inactive for 200.40s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1665473.467017] Pid: 85918, comm: ll_ost00_007 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665473.477188] Call Trace: [1665473.479816] [] 0xffffffffffffffff [1665473.485021] LustreError: dumping log to /tmp/lustre-log.1549443190.85918 [1665474.717765] LNet: Service thread pid 79103 was inactive for 201.68s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1665474.734871] Pid: 79103, comm: ll_ost00_008 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665474.745043] Call Trace: [1665474.747671] [] 0xffffffffffffffff [1665474.752879] Pid: 82406, comm: ll_ost00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665474.763044] Call Trace: [1665474.765672] [] 0xffffffffffffffff [1665474.770846] Pid: 82860, comm: ll_ost00_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665474.781019] Call Trace: [1665474.783646] [] 0xffffffffffffffff [1665513.047909] Lustre: 82406:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443229/real 1549443229] req@ffff90abd3e6d100 x1623662775085808/t0(0) o106->fir-OST0001@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443240 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665513.075680] Lustre: 82406:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 26450921 previous similar messages [1665528.538304] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665528.549085] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 26497084 previous similar messages [1665742.961684] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.0.10.3@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665768.060801] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1665768.071233] Lustre: Skipped 202 previous similar messages [1665769.054464] Lustre: 85918:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443485/real 1549443485] req@ffff90a248ef1200 x1623662775085776/t0(0) o106->fir-OST0007@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443496 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665769.082255] Lustre: 85918:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 52898261 previous similar messages [1665784.544859] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665784.555655] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 52902014 previous similar messages [1665793.138787] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.0.10.3@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665867.299997] Lustre: 82405:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff90abf7cb5c50 x1623967095656976/t0(0) o101->0ccdc4e2-9749-c9a5-afb4-85874ce74d6c@10.0.10.3@o2ib7:124/0 lens 344/400 e 24 to 0 dl 1549443589 ref 2 fl Interpret:/0/0 rc 0/0 [1665873.621858] Lustre: fir-OST0003: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1665873.632028] Lustre: Skipped 5 previous similar messages [1666222.605968] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.9.102.22@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1666281.067579] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443997/real 1549443997] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549444008 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1666281.095359] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 105914443 previous similar messages [1666296.557962] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1666296.568744] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 105899475 previous similar messages [1666399.721868] Lustre: fir-OST0001: Connection restored to b5581c4c-47a8-b046-19f4-e572468b3106 (at 10.9.102.32@o2ib4) [1666399.732480] Lustre: Skipped 64 previous similar messages [1666475.356406] Lustre: fir-OST0003: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1666475.356407] Lustre: fir-OST0001: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1666475.356411] Lustre: Skipped 2 previous similar messages [1666475.382151] Lustre: Skipped 2 previous similar messages [1666753.102287] Lustre: fir-OST0003: Client d405c92c-c32a-78c2-b321-55f4db7bb864 (at 10.9.102.14@o2ib4) reconnecting [1666841.029856] Lustre: fir-OST000b: Export ffff90bbf9ac0400 already connecting from 10.8.8.22@o2ib6 [1666881.082935] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549444597/real 1549444597] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549444608 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1666881.110702] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 123691589 previous similar messages [1666891.192122] Lustre: fir-OST000b: Client 67a91875-0062-0ad3-9302-835366d40903 (at 10.8.8.22@o2ib6) reconnecting [1666896.573323] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1666896.584103] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 123784463 previous similar messages [1666928.144441] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.0.10.3@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1667032.114116] Lustre: fir-OST0007: Connection restored to d35217f0-5ade-6000-33a4-4fb011efe6c5 (at 10.8.4.8@o2ib6) [1667032.114118] Lustre: fir-OST0007: Connection restored to bcafe9e2-1bfd-c9dc-9b3b-f32359a46a76 (at 10.8.3.8@o2ib6) [1667032.114121] Lustre: Skipped 113 previous similar messages [1667032.140386] Lustre: Skipped 3 previous similar messages [1667075.588140] Lustre: fir-OST0001: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1667075.598313] Lustre: Skipped 6 previous similar messages [1667350.728795] Lustre: fir-OST000b: Client 7936d72a-098c-6dfd-6893-32b658b77868 (at 10.9.103.9@o2ib4) reconnecting [1667350.739059] Lustre: Skipped 1 previous similar message [1667481.098291] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549445197/real 1549445197] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549445208 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1667481.126057] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 126715367 previous similar messages [1667496.588685] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1667496.599469] LNetError: 85918:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 126696441 previous similar messages [1667635.296137] Lustre: fir-OST0001: Connection restored to a6388386-fc78-2645-7cfe-6de1bc9e10a6 (at 10.9.102.37@o2ib4) [1667635.306745] Lustre: Skipped 137 previous similar messages [1667935.727462] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.9.104.65@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668000.288417] Lustre: fir-OST0007: Client 948f385b-b60e-f471-69e7-ab8aac33a9fb (at 10.8.4.7@o2ib6) reconnecting [1668000.298510] Lustre: Skipped 12 previous similar messages [1668081.113639] Lustre: 82406:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549445797/real 1549445797] req@ffff90abd3e6d100 x1623662775085808/t0(0) o106->fir-OST0001@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549445808 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1668081.141412] Lustre: 82406:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 125086875 previous similar messages [1668096.604041] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1668096.614825] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 125119449 previous similar messages [1668207.635948] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.8.7.23@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668207.653313] LustreError: Skipped 1 previous similar message [1668215.100402] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.8.10.1@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668218.284181] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.8.7.17@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668236.083210] Lustre: fir-OST000b: Connection restored to cec68a9e-fc42-8b83-b21d-285fcd29817f (at 10.8.7.16@o2ib6) [1668236.093641] Lustre: Skipped 400 previous similar messages [1668238.918918] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.8.10.27@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668238.936382] LustreError: Skipped 8 previous similar messages [1668266.192440] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.8.13@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668266.209812] LustreError: Skipped 3 previous similar messages [1668299.449574] LustreError: 137-5: fir-OST0006_UUID: not available for connect from 10.8.6.33@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668299.466941] LustreError: Skipped 6 previous similar messages [1668369.122201] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.8.8.7@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668369.139484] LustreError: Skipped 2 previous similar messages [1668433.181411] LustreError: 137-5: fir-OST0006_UUID: not available for connect from 10.9.102.72@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668433.198953] LustreError: Skipped 1 previous similar message [1668582.672688] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.8.7.14@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668582.690059] LustreError: Skipped 1 previous similar message [1668607.761253] Lustre: fir-OST0003: Client a9e2a338-60d5-67b8-b23e-aaade58467c0 (at 10.8.7.14@o2ib6) reconnecting [1668607.771427] Lustre: Skipped 83 previous similar messages [1668681.128999] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549446397/real 1549446397] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549446408 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1668681.156787] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 128102352 previous similar messages [1668696.619392] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1668696.630170] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 128189191 previous similar messages [1668836.099920] Lustre: fir-OST0005: Connection restored to 645e23fe-e756-9b3d-2aec-968de2480e79 (at 10.8.7.4@o2ib6) [1668836.110267] Lustre: Skipped 816 previous similar messages [1668886.288929] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.8.11.24@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668886.306385] LustreError: Skipped 32 previous similar messages [1669275.255939] Lustre: fir-OST0003: haven't heard from client b1ae38e6-4e9c-51ec-0d30-af325c63f3ef (at 10.8.4.15@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90af85df7800, cur 1549446992 expire 1549446842 last 1549446765 [1669276.144744] Lustre: fir-OST000b: haven't heard from client b1ae38e6-4e9c-51ec-0d30-af325c63f3ef (at 10.8.4.15@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90af85df7c00, cur 1549446992 expire 1549446842 last 1549446765 [1669276.166530] Lustre: Skipped 1 previous similar message [1669281.144363] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549446997/real 1549446997] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549447008 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1669281.172162] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 127983260 previous similar messages [1669296.634754] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1669296.645553] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 127897045 previous similar messages [1669297.835865] Lustre: fir-OST0003: Client a4f2c4c1-03c5-819a-6852-1875c7d76a33 (at 10.8.19.7@o2ib6) reconnecting [1669297.846032] Lustre: Skipped 31 previous similar messages [1669402.256024] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.4.15@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1669402.273390] LustreError: Skipped 51 previous similar messages [1669441.551881] Lustre: fir-OST0003: Connection restored to 56a8fa87-5a1b-6c27-dd37-3aed2bd4a561 (at 10.8.6.9@o2ib6) [1669441.562230] Lustre: Skipped 388 previous similar messages [1669881.159713] Lustre: 82406:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549447597/real 1549447597] req@ffff90abd3e6d100 x1623662775085808/t0(0) o106->fir-OST0001@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549447608 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1669881.187482] Lustre: 82406:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 127643027 previous similar messages [1669896.650117] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1669896.660895] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 127659201 previous similar messages [1669923.236236] Lustre: fir-OST0001: Client e527b419-86b9-ff93-48e0-b15e55994667 (at 10.9.106.28@o2ib4) reconnecting [1669923.246599] Lustre: Skipped 18 previous similar messages [1670022.337798] LustreError: 137-5: fir-OST0006_UUID: not available for connect from 10.8.4.24@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1670022.355165] LustreError: Skipped 9 previous similar messages [1670046.409479] Lustre: fir-OST0003: Connection restored to e2c4a480-f1d1-2209-d809-5088ddc9ced3 (at 10.8.4.24@o2ib6) [1670046.419916] Lustre: Skipped 364 previous similar messages [1670481.175069] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549448197/real 1549448197] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549448208 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1670481.202832] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 127493931 previous similar messages [1670496.665471] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1670496.676250] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 127638521 previous similar messages [1670657.876393] Lustre: fir-OST0001: Connection restored to 848db794-00ba-ef76-c052-86987458d4b8 (at 10.9.104.55@o2ib4) [1670657.887028] Lustre: Skipped 120 previous similar messages [1670682.465051] Lustre: fir-OST0003: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1670682.475226] Lustre: Skipped 13 previous similar messages [1671081.190439] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549448797/real 1549448797] req@ffff90a048d53900 x1623662775085760/t0(0) o106->fir-OST0005@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549448808 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1671081.218227] Lustre: 79103:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 129058471 previous similar messages [1671096.680824] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1671096.691601] LNetError: 79103:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 128989575 previous similar messages [1671212.050688] LNet: Service thread pid 79103 completed after 5938.86s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [1671212.067105] LNet: Skipped 3 previous similar messages [1671266.739664] Lustre: fir-OST0001: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1671266.750193] Lustre: Skipped 90 previous similar messages [1671929.722108] Lustre: fir-OST0009: Connection restored to d3157baf-de90-86ed-7c87-5f5f5c909a71 (at 10.9.105.15@o2ib4) [1671929.732724] Lustre: Skipped 60 previous similar messages [1672530.657405] Lustre: fir-OST0003: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1672530.667930] Lustre: Skipped 142 previous similar messages [1673176.171844] Lustre: fir-OST000b: Connection restored to 8d1694af-d32c-3288-00b1-fe38a72175fb (at 10.9.104.37@o2ib4) [1673176.182459] Lustre: Skipped 63 previous similar messages [1673777.248197] Lustre: fir-OST0003: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1673777.258721] Lustre: Skipped 131 previous similar messages [1674384.306641] Lustre: fir-OST0007: Connection restored to 6ff78f59-4d7f-d121-1724-154727e4138f (at 10.9.0.64@o2ib4) [1674384.317073] Lustre: Skipped 118 previous similar messages [1674986.324070] Lustre: fir-OST0003: Connection restored to ee310e5c-55e4-0a0f-af99-ae0536891531 (at 10.8.27.2@o2ib6) [1674986.334531] Lustre: Skipped 226 previous similar messages [1675588.402230] Lustre: fir-OST0003: Connection restored to ee310e5c-55e4-0a0f-af99-ae0536891531 (at 10.8.27.2@o2ib6) [1675588.412674] Lustre: Skipped 28 previous similar messages [1676207.381086] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1676207.391521] Lustre: Skipped 29 previous similar messages [1676835.874819] Lustre: fir-OST0007: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1676835.885345] Lustre: Skipped 33 previous similar messages [1677509.924391] Lustre: fir-OST000b: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1677509.935070] Lustre: Skipped 25 previous similar messages [1678124.493235] Lustre: fir-OST0003: Connection restored to 7936d72a-098c-6dfd-6893-32b658b77868 (at 10.9.103.9@o2ib4) [1678124.503770] Lustre: Skipped 59 previous similar messages [1678764.995099] Lustre: fir-OST0003: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1678765.005616] Lustre: Skipped 24 previous similar messages [1679140.443320] Lustre: fir-OST0001: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1679140.452454] Lustre: Skipped 6 previous similar messages [1679141.014422] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:470047 to 0xb40000402:470081 [1679141.022733] Lustre: 103468:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549456850/real 1549456850] req@ffff90ca425d2a00 x1623662777184368/t0(0) o104->fir-OST000b@10.9.105.65@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549456857 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1679141.022736] Lustre: 103468:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 28160227 previous similar messages [1679456.628603] Lustre: fir-OST0001: Connection restored to cef0c972-4cf2-c1ae-1fb9-c404ed41cb0f (at 10.8.8.22@o2ib6) [1679456.639045] Lustre: Skipped 37 previous similar messages [1680092.488856] Lustre: fir-OST0007: Connection restored to b5581c4c-47a8-b046-19f4-e572468b3106 (at 10.9.102.32@o2ib4) [1680092.499468] Lustre: Skipped 27 previous similar messages [1680699.048052] Lustre: fir-OST0003: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1680699.058579] Lustre: Skipped 57 previous similar messages [1681307.499090] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1681307.509526] Lustre: Skipped 41 previous similar messages [1681907.516661] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1681907.527102] Lustre: Skipped 22 previous similar messages [1682508.529297] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1682508.539773] Lustre: Skipped 29 previous similar messages [1683108.561719] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1683108.572175] Lustre: Skipped 42 previous similar messages [1683708.577645] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1683708.588094] Lustre: Skipped 24 previous similar messages [1684311.629126] Lustre: fir-OST0003: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1684311.639660] Lustre: Skipped 33 previous similar messages [1684967.163511] Lustre: fir-OST0003: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1684967.174032] Lustre: Skipped 28 previous similar messages [1685577.113697] Lustre: fir-OST0007: Connection restored to 3608e712-e3ad-8999-51dd-65547ce62d69 (at 10.8.7.25@o2ib6) [1685577.124133] Lustre: Skipped 483 previous similar messages [1686183.106261] Lustre: fir-OST0007: Connection restored to 8d89fca0-9472-dc06-65e2-fd0a61adf564 (at 10.9.106.23@o2ib4) [1686183.116874] Lustre: Skipped 246 previous similar messages [1686483.662221] Lustre: fir-OST0007: haven't heard from client ef658cd6-a6d2-3958-b89c-2c53bab505a2 (at 10.8.27.35@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90acd285a400, cur 1549464200 expire 1549464050 last 1549463973 [1686483.684106] Lustre: Skipped 3 previous similar messages [1686785.609447] Lustre: fir-OST0007: Connection restored to ce7c2edb-40f8-3a94-8d08-7f35c7e4a9ee (at 10.9.101.3@o2ib4) [1686785.619969] Lustre: Skipped 190 previous similar messages [1686788.673595] Lustre: fir-OST0005: haven't heard from client ee310e5c-55e4-0a0f-af99-ae0536891531 (at 10.8.27.2@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90bb90f8c000, cur 1549464505 expire 1549464355 last 1549464278 [1686788.695419] Lustre: Skipped 5 previous similar messages [1687386.153759] Lustre: fir-OST0007: Connection restored to 657c4193-574d-7ac0-e927-0684806e81d2 (at 10.9.101.42@o2ib4) [1687386.164367] Lustre: Skipped 323 previous similar messages [1687987.764659] Lustre: fir-OST0005: Connection restored to c2fa3877-6bf6-c6b9-4fe2-c5a65c4724d7 (at 10.9.102.49@o2ib4) [1687987.775287] Lustre: Skipped 178 previous similar messages [1688587.832207] Lustre: fir-OST0001: Connection restored to ee310e5c-55e4-0a0f-af99-ae0536891531 (at 10.8.27.2@o2ib6) [1688587.842650] Lustre: Skipped 311 previous similar messages [1689196.801625] Lustre: fir-OST0005: Connection restored to 5b322ed5-1f83-3d4a-541c-ca479ed4d108 (at 10.8.18.18@o2ib6) [1689196.812150] Lustre: Skipped 150 previous similar messages [1689797.374410] Lustre: fir-OST0009: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1689797.384935] Lustre: Skipped 107 previous similar messages [1689990.346229] Lustre: fir-OST0005: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1689990.355377] Lustre: Skipped 2 previous similar messages [1690001.084601] Lustre: fir-OST0001: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1690001.093735] Lustre: Skipped 1 previous similar message [1690025.436326] LNetError: 79895:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1690025.446494] LNetError: 79895:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Skipped 7 previous similar messages [1690025.456836] LNetError: 79895:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (5): c: 0, oc: 0, rc: 8 [1690025.468905] LNetError: 79895:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Skipped 7 previous similar messages [1690025.479329] Lustre: 79954:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549467736/real 1549467741] req@ffff90c244042a00 x1623662780269232/t0(0) o400->fir-MDT0000-lwp-OST0009@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549467968 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1690025.479370] Lustre: fir-MDT0002-lwp-OST0009: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1690025.479373] Lustre: Skipped 2 previous similar messages [1690025.479420] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1690025.543786] Lustre: 79954:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [1690027.436371] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 2 seconds [1690027.446624] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1690027.456180] Lustre: fir-MDT0000-lwp-OST0005: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1690027.472362] Lustre: Skipped 9 previous similar messages [1690033.436522] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1690033.446777] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 14 previous similar messages [1690058.437166] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1690058.447452] Lustre: 79952:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549467768/real 1549467774] req@ffff90ca425d7200 x1623662780278352/t0(0) o400->fir-MDT0002-lwp-OST0007@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549468000 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1690058.476364] Lustre: 79952:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1690058.486285] Lustre: fir-MDT0002-lwp-OST0007: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1690065.437351] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [1690085.437856] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 23 seconds [1690114.438596] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1690114.448852] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 11 previous similar messages [1690155.191412] Lustre: fir-OST000b: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1690155.191544] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1690155.191546] LustreError: Skipped 7 previous similar messages [1690155.191736] Lustre: fir-OST0009: deleting orphan objects from 0x0:677667 to 0x0:677697 [1690155.192656] Lustre: fir-OST0001: deleting orphan objects from 0x0:677623 to 0x0:677665 [1690155.192678] Lustre: fir-OST0003: deleting orphan objects from 0x0:677571 to 0x0:677601 [1690155.193646] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:472311 to 0x540000400:472385 [1690155.194144] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:472275 to 0xb40000402:472321 [1690155.194241] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:472202 to 0xbc0000402:472289 [1690155.276583] Lustre: Skipped 6 previous similar messages [1690155.282157] Lustre: fir-OST000b: deleting orphan objects from 0x0:677657 to 0x0:677697 [1690180.280680] Lustre: fir-OST0005: deleting orphan objects from 0x0:677598 to 0x0:677633 [1690180.281088] Lustre: fir-OST0007: deleting orphan objects from 0x0:677604 to 0x0:677633 [1690180.281671] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:472284 to 0x380000400:472321 [1690180.281936] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:472260 to 0x800000400:472353 [1690180.282046] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:472222 to 0xb00000402:472257 [1690252.020131] Lustre: 79952:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549467736/real 1549467736] req@ffff90c244043000 x1623662780269024/t0(0) o400->fir-MDT0002-lwp-OST0007@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549467968 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1690400.962736] Lustre: fir-OST0007: Connection restored to afd89a08-6b67-04f7-2303-d504e382d3cd (at 10.9.105.61@o2ib4) [1690400.973346] Lustre: Skipped 183 previous similar messages [1690897.726662] LNetError: 103472:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.15@tcp from [1690897.737535] LNetError: 103472:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 24819154 previous similar messages [1690897.748855] Lustre: 103472:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468613/real 1549468613] req@ffff90c0d7667800 x1623662780614784/t0(0) o104->fir-OST000b@10.10.113.15@tcp:15/16 lens 296/224 e 0 to 1 dl 1549468624 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [1690897.776713] Lustre: 103472:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6930 previous similar messages [1690972.729557] LNetError: 103476:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.15@tcp from [1690972.740427] LNetError: 103476:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 18531005 previous similar messages [1690972.751733] Lustre: 103476:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468688/real 1549468688] req@ffff90c1ef720900 x1623662780614864/t0(0) o104->fir-OST0001@10.10.113.15@tcp:15/16 lens 296/224 e 0 to 1 dl 1549468699 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1690972.779630] Lustre: 103476:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 18526483 previous similar messages [1691008.562039] Lustre: fir-OST0009: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1691008.572715] Lustre: Skipped 107 previous similar messages [1691048.121504] LustreError: 56691:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) ### client (nid 10.10.113.15@tcp) failed to reply to blocking AST (req@ffff90ca6a280600 x1623662780614912 status 0 rc -110), evict it ns: filter-fir-OST0009_UUID lock: ffff90b2970fc800/0xda500c83d50010e3 lrc: 4/0,0 mode: PR/PR res: [0x540000400:0x72c79:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 17179869184->18446744073709551615) flags: 0x60000400010020 nid: 10.10.113.15@tcp remote: 0xdf1b55af52e16750 expref: 7 pid: 42300 timeout: 1691153 lvb_type: 1 [1691048.121512] LustreError: 138-a: fir-OST0003: A client on nid 10.10.113.15@tcp was evicted due to a lock blocking callback time out: rc -110 [1691048.121536] LustreError: 81679:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 151s: evicting client at 10.10.113.15@tcp ns: filter-fir-OST000b_UUID lock: ffff90b2970fc380/0xda500c83d50010dc lrc: 3/0,0 mode: PR/PR res: [0x380000400:0x72c56:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 17179869184->18446744073709551615) flags: 0x60000400010020 nid: 10.10.113.15@tcp remote: 0xdf1b55af52e16718 expref: 7 pid: 42300 timeout: 0 lvb_type: 1 [1691048.121538] LustreError: 81679:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 1 previous similar message [1691048.233965] LustreError: 56691:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) Skipped 4 previous similar messages [1691048.736416] LustreError: 81679:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 151s: evicting client at 10.10.113.15@tcp ns: filter-fir-OST0009_UUID lock: ffff90b2970fc800/0xda500c83d50010e3 lrc: 3/0,0 mode: PR/PR res: [0x540000400:0x72c79:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 17179869184->18446744073709551615) flags: 0x60000400010020 nid: 10.10.113.15@tcp remote: 0xdf1b55af52e16750 expref: 8 pid: 42300 timeout: 0 lvb_type: 1 [1691048.778686] LustreError: 81679:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 2 previous similar messages [1691117.957424] Lustre: fir-OST0003: Client 95f41e85-91f0-6aee-d80e-dc57a1b83666 (at 10.10.113.15@tcp) reconnecting [1691117.967688] Lustre: Skipped 5 previous similar messages [1691122.733398] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1691122.744110] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 29098359 previous similar messages [1691122.755409] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468838/real 1549468838] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549468849 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1691122.783088] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 29090539 previous similar messages [1691281.311018] Lustre: fir-OST000b: Client 5ba2142c-879f-ac28-9d4a-a3788afebea0 (at 10.8.7.35@o2ib6) reconnecting [1691315.661549] Lustre: fir-OST0007: Client d3157baf-de90-86ed-7c87-5f5f5c909a71 (at 10.9.105.15@o2ib4) reconnecting [1691370.287283] Lustre: fir-OST000b: Client 3a10c79f-70d6-d387-a9b4-ab231a26d59b (at 10.9.102.24@o2ib4) reconnecting [1691410.357051] Lustre: fir-OST000b: Client 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) reconnecting [1691422.741079] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1691422.751796] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 35852571 previous similar messages [1691422.763075] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549469138/real 1549469138] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549469149 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1691422.790778] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 35852572 previous similar messages [1691514.508638] Lustre: fir-OST0001: Client e8e9feb6-5cc2-22ca-f250-c7608b5324bd (at 10.8.23.19@o2ib6) reconnecting [1691514.518904] Lustre: Skipped 3 previous similar messages [1691608.731245] Lustre: fir-OST0001: Connection restored to 4376dd10-0b2f-912c-8c73-35fe5309aa36 (at 10.8.22.25@o2ib6) [1691608.741770] Lustre: Skipped 791 previous similar messages [1691620.543153] Lustre: 84270:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff90cbfbc19850 x1624699509228496/t0(0) o103->d5e4b60f-fe33-b991-7d48-5b8db7e07ab0@10.8.0.67@o2ib6:206/0 lens 328/224 e 24 to 0 dl 1549469341 ref 2 fl Interpret:H/0/0 rc 0/0 [1691620.572306] Lustre: 84270:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 3 previous similar messages [1691627.591325] LNet: Service thread pid 81672 was inactive for 601.88s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1691627.608438] LNet: Skipped 2 previous similar messages [1691627.613673] Pid: 81672, comm: ldlm_cn02_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1691627.623948] Call Trace: [1691627.626584] [] 0xffffffffffffffff [1691627.631786] LustreError: dumping log to /tmp/lustre-log.1549469343.81672 [1691667.701634] Lustre: fir-OST000b: Client 5bd1312d-d4d9-79f2-e6c0-fb1a0e35eed8 (at 10.8.12.17@o2ib6) reconnecting [1691667.711895] Lustre: Skipped 81 previous similar messages [1691797.998780] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:475784 to 0x380000400:476065 [1691837.423121] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:475910 to 0x800000400:476193 [1691890.484059] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1691890.494312] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 11 previous similar messages [1691890.503916] Lustre: fir-MDT0002-lwp-OST0001: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1691890.520084] Lustre: Skipped 6 previous similar messages [1691891.484127] Lustre: fir-MDT0000-lwp-OST0005: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1691891.500285] Lustre: Skipped 4 previous similar messages [1691894.485162] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 20 seconds [1691894.495501] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 20 previous similar messages [1691895.484215] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1691967.212622] Lustre: fir-OST0005: deleting orphan objects from 0x0:677645 to 0x0:677665 [1691967.212625] Lustre: fir-OST000b: deleting orphan objects from 0x0:677709 to 0x0:677729 [1691967.212790] Lustre: fir-OST0007: deleting orphan objects from 0x0:677645 to 0x0:677665 [1691967.212974] Lustre: fir-OST0003: deleting orphan objects from 0x0:677612 to 0x0:677633 [1691967.213099] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:476068 to 0x380000400:476097 [1691967.213163] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:475959 to 0x540000400:476225 [1691967.213224] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:476196 to 0x800000400:476225 [1691967.213294] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:475829 to 0xb00000402:476065 [1691967.213345] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:475898 to 0xb40000402:476545 [1691967.213398] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:475851 to 0xbc0000402:476065 [1691967.213470] Lustre: fir-OST0009: deleting orphan objects from 0x0:677709 to 0x0:677729 [1691993.564065] Lustre: fir-OST0001: Client 850f5687-1489-7b0d-073f-8431f2aebf8a (at 10.8.20.17@o2ib6) reconnecting [1691993.574326] Lustre: Skipped 39 previous similar messages [1692022.756436] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1692022.767127] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 71955677 previous similar messages [1692022.778436] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549469738/real 1549469738] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549469749 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1692022.806112] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 71955691 previous similar messages [1692213.187673] Lustre: fir-OST0001: Connection restored to 45a12f91-6aa3-f0ae-a299-8aadf4c776a5 (at 10.8.17.5@o2ib6) [1692213.198103] Lustre: Skipped 717 previous similar messages [1692601.362809] Lustre: fir-OST0001: Client 2d862fb5-44ea-7d60-cb0c-d7ea36856c36 (at 10.8.26.9@o2ib6) reconnecting [1692601.372982] Lustre: Skipped 78 previous similar messages [1692622.771791] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1692622.782488] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 71983267 previous similar messages [1692622.793798] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549470338/real 1549470338] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549470349 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1692622.821494] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 71983270 previous similar messages [1692828.761207] Lustre: fir-OST000b: Connection restored to d5e4b60f-fe33-b991-7d48-5b8db7e07ab0 (at 10.8.0.67@o2ib6) [1692828.771658] Lustre: Skipped 554 previous similar messages [1692838.632413] Lustre: fir-OST0001: deleting orphan objects from 0x0:677676 to 0x0:677697 [1693207.438585] Lustre: fir-OST000b: Client 935e7eb3-1ff6-7dda-ab9a-d14a4b5f1855 (at 10.9.103.32@o2ib4) reconnecting [1693207.448929] Lustre: Skipped 11 previous similar messages [1693222.787149] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1693222.797859] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 71978939 previous similar messages [1693222.809149] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549470938/real 1549470938] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549470949 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1693222.836851] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 71978937 previous similar messages [1693429.866090] Lustre: fir-OST000b: Connection restored to d5e4b60f-fe33-b991-7d48-5b8db7e07ab0 (at 10.8.0.67@o2ib6) [1693429.876524] Lustre: Skipped 201 previous similar messages [1693822.802509] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1693822.813202] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72108758 previous similar messages [1693822.824504] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549471538/real 1549471538] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549471549 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1693822.852212] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72108759 previous similar messages [1693857.610591] Lustre: fir-OST0009: Client 40ad8941-e56d-16a8-06a4-9f03d85f8e6b (at 10.9.107.64@o2ib4) reconnecting [1693857.620941] Lustre: Skipped 6 previous similar messages [1694030.968707] Lustre: fir-OST000b: Connection restored to d5e4b60f-fe33-b991-7d48-5b8db7e07ab0 (at 10.8.0.67@o2ib6) [1694030.979141] Lustre: Skipped 253 previous similar messages [1694170.887327] Lustre: fir-OST0009: haven't heard from client 40ad8941-e56d-16a8-06a4-9f03d85f8e6b (at 10.9.107.64@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90bb90f89400, cur 1549471887 expire 1549471737 last 1549471660 [1694170.909293] Lustre: Skipped 1 previous similar message [1694422.817866] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1694422.828565] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72006640 previous similar messages [1694422.839868] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549472138/real 1549472138] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549472149 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1694422.867549] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72006640 previous similar messages [1694464.341657] Lustre: fir-OST0005: Client a37c418b-5455-262c-ca39-fe09a1d64b6a (at 10.9.106.7@o2ib4) reconnecting [1694464.351924] Lustre: Skipped 19 previous similar messages [1694631.176379] Lustre: fir-OST0001: Connection restored to f97f4802-8c0b-e9cd-e8d7-93692decf22a (at 10.9.102.8@o2ib4) [1694631.186902] Lustre: Skipped 348 previous similar messages [1695022.833217] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1695022.843919] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72138542 previous similar messages [1695022.855220] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549472738/real 1549472738] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549472749 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1695022.882920] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72138539 previous similar messages [1695132.271499] Lustre: fir-OST0001: Client 1127e5ab-6d36-a097-38c2-cf0f21f5ec17 (at 10.8.26.8@o2ib6) reconnecting [1695132.281670] Lustre: Skipped 17 previous similar messages [1695233.278902] Lustre: fir-OST000b: Connection restored to d5e4b60f-fe33-b991-7d48-5b8db7e07ab0 (at 10.8.0.67@o2ib6) [1695233.289340] Lustre: Skipped 378 previous similar messages [1695494.295942] Lustre: fir-OST0007: deleting orphan objects from 0xb00000401:134425 to 0xb00000401:134465 [1695516.611314] Lustre: fir-OST0009: deleting orphan objects from 0x540000402:134511 to 0x540000402:134561 [1695524.693044] Lustre: fir-OST0005: Export ffff90cbf876dc00 already connecting from 10.8.20.15@o2ib6 [1695524.693045] Lustre: fir-OST0009: Export ffff90cbf99cb400 already connecting from 10.8.20.15@o2ib6 [1695597.898847] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.9.106.6@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1695597.916324] LustreError: Skipped 4 previous similar messages [1695622.848579] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1695622.859273] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72234268 previous similar messages [1695622.870575] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549473338/real 1549473338] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549473349 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1695622.898259] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72234271 previous similar messages [1695689.922883] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1695689.940335] LustreError: Skipped 10 previous similar messages [1695733.467042] Lustre: fir-OST0007: Client ef406a33-dbdc-c381-15c5-4fb662abecc1 (at 10.8.27.10@o2ib6) reconnecting [1695733.477301] Lustre: Skipped 34 previous similar messages [1695833.463673] Lustre: fir-OST0007: Connection restored to 1a875747-e8ac-b714-1ac8-5e36c1c83f8b (at 10.8.30.15@o2ib6) [1695833.474194] Lustre: Skipped 531 previous similar messages [1696222.863936] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1696222.874643] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72311963 previous similar messages [1696222.885933] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549473938/real 1549473938] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549473949 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1696222.913626] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72311959 previous similar messages [1696355.853698] Lustre: fir-OST0001: Client 2b859be6-67db-6942-628b-a867997f44c5 (at 10.8.2.10@o2ib6) reconnecting [1696355.863878] Lustre: Skipped 26 previous similar messages [1696435.763983] Lustre: fir-OST0005: Connection restored to f88d3e4f-b8ad-7e3f-e052-b857e571de2a (at 10.9.107.13@o2ib4) [1696435.774592] Lustre: Skipped 322 previous similar messages [1696720.205187] Lustre: fir-OST0005: Export ffff90c42842b800 already connecting from 10.8.8.22@o2ib6 [1696822.879297] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1696822.889992] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72283944 previous similar messages [1696822.901290] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549474538/real 1549474538] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549474549 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1696822.928995] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72283947 previous similar messages [1696969.987888] Lustre: fir-OST000b: Client 67525666-a063-cbc2-2269-3d7cda07ffe3 (at 10.9.0.81@o2ib4) reconnecting [1696969.998062] Lustre: Skipped 12 previous similar messages [1697036.758584] Lustre: fir-OST0001: Connection restored to c50e9e63-bc69-ffb4-d9c5-0a1d77a8b849 (at 10.9.106.60@o2ib4) [1697036.769190] Lustre: Skipped 273 previous similar messages [1697422.894655] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1697422.905353] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72117052 previous similar messages [1697422.916648] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549475138/real 1549475138] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549475149 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1697422.944345] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72117051 previous similar messages [1697574.872085] Lustre: fir-OST0009: Client 9114d135-37ef-ec55-8f47-397cb880c459 (at 10.8.28.6@o2ib6) reconnecting [1697574.882256] Lustre: Skipped 24 previous similar messages [1697639.807565] Lustre: fir-OST000b: Connection restored to d5e4b60f-fe33-b991-7d48-5b8db7e07ab0 (at 10.8.0.67@o2ib6) [1697639.817994] Lustre: Skipped 426 previous similar messages [1698022.910012] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.6@tcp from [1698022.920706] LNetError: 81672:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 72130516 previous similar messages [1698022.932008] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549475738/real 1549475738] req@ffff90c0d7667b00 x1623662780706176/t0(0) o105->fir-OST000b@10.10.113.6@tcp:15/16 lens 360/224 e 0 to 1 dl 1549475749 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1698022.959705] Lustre: 81672:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 72130517 previous similar messages [1698200.290210] Lustre: fir-OST000b: Client cbf5d06d-ba23-f67d-ac30-844ab79b193e (at 10.9.103.29@o2ib4) reconnecting [1698200.300555] Lustre: Skipped 24 previous similar messages [1698241.372782] Lustre: fir-OST000b: Connection restored to d5e4b60f-fe33-b991-7d48-5b8db7e07ab0 (at 10.8.0.67@o2ib6) [1698241.383216] Lustre: Skipped 406 previous similar messages [1698567.984438] Lustre: fir-OST0007: haven't heard from client 5fe1f35d-2720-6130-4811-b6c831015c6a (at 10.8.20.15@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90ba027d7800, cur 1549476284 expire 1549476134 last 1549476057 [1698572.909423] Lustre: fir-OST000b: haven't heard from client db7578a7-294b-6d6d-61e8-bb06998c6482 (at 10.10.113.6@tcp) in 216 seconds. I think it's dead, and I am evicting it. exp ffff90af53ed4000, cur 1549476288 expire 1549476134 last 1549476072 [1698572.931219] Lustre: Skipped 1 previous similar message [1698572.936644] LNet: Service thread pid 81672 completed after 7547.04s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [1698845.487221] Lustre: fir-OST000b: Connection restored to 56dfdd02-4757-479f-42af-72ec5af15e3f (at 10.8.8.18@o2ib6) [1698845.497660] Lustre: Skipped 1067 previous similar messages [1699379.993759] Lustre: fir-OST0001: haven't heard from client 57eeb417-c604-9e85-88ff-67a98be0814e (at 10.8.10.33@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90b87dfd7c00, cur 1549477096 expire 1549476946 last 1549476869 [1699380.015669] Lustre: Skipped 4 previous similar messages [1699446.048641] Lustre: fir-OST0003: Connection restored to 5b896ca8-2947-8e31-025e-233ef4d66e00 (at 10.8.17.11@o2ib6) [1699446.059163] Lustre: Skipped 1129 previous similar messages [1699493.801889] Lustre: fir-OST0009: Client 7e4f58d8-6e8c-32f9-59fb-7de9af0df4b4 (at 10.9.106.12@o2ib4) reconnecting [1699493.812237] Lustre: Skipped 32 previous similar messages [1699496.209751] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000401:135492 to 0xbc0000401:135553 [1699496.268440] Lustre: fir-OST0005: deleting orphan objects from 0x800000401:147290 to 0x800000401:147361 [1699497.125783] Lustre: fir-OST000b: deleting orphan objects from 0x380000401:147308 to 0x380000401:147393 [1699541.004129] Lustre: fir-OST0007: deleting orphan objects from 0xb00000401:135504 to 0xb00000401:135521 [1699579.514141] Lustre: fir-OST000b: Client 7e4f58d8-6e8c-32f9-59fb-7de9af0df4b4 (at 10.9.106.12@o2ib4) reconnecting [1699579.524500] Lustre: Skipped 9 previous similar messages [1699647.088050] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1699672.166728] Lustre: fir-OST0007: deleting orphan objects from 0xb00000401:135526 to 0xb00000401:135553 [1700046.605967] Lustre: fir-OST000b: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [1700046.616407] Lustre: Skipped 1160 previous similar messages [1700048.007593] Lustre: fir-OST000b: haven't heard from client a13f0f5a-2377-84a5-980d-206962591c11 (at 10.8.11.10@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90cbfa414000, cur 1549477764 expire 1549477614 last 1549477537 [1700048.029499] Lustre: Skipped 2 previous similar messages [1700063.009100] Lustre: fir-OST0005: haven't heard from client e123259a-9e5e-2357-824d-ff78a3eedca5 (at 10.8.11.9@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90af53ed1400, cur 1549477779 expire 1549477629 last 1549477552 [1700063.030893] Lustre: Skipped 6 previous similar messages [1700496.405604] Lustre: fir-OST0009: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1700496.414739] Lustre: Skipped 11 previous similar messages [1700498.018096] Lustre: fir-OST0009: deleting orphan objects from 0x540000401:147424 to 0x540000401:147489 [1700500.980283] Lustre: fir-MDT0002-lwp-OST0007: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1700500.996441] Lustre: Skipped 9 previous similar messages [1700501.285943] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:478007 to 0x800000400:478177 [1700506.758285] Lustre: fir-OST0005: deleting orphan objects from 0x800000401:147471 to 0x800000401:147489 [1700510.464576] Lustre: fir-OST0001: deleting orphan objects from 0xb40000400:147440 to 0xb40000400:147521 [1700576.706386] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 70 seconds [1700576.716727] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 4 previous similar messages [1700579.706469] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700579.716724] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [1700582.706550] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700582.716800] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1700601.708021] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 14 seconds [1700601.718367] LNet: 79895:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1700604.361413] Lustre: Failing over fir-OST0001 [1700604.365867] Lustre: Skipped 3 previous similar messages [1700604.449912] Lustre: fir-OST0001: Not available for connect from 10.8.21.20@o2ib6 (stopping) [1700604.458441] Lustre: Skipped 1 previous similar message [1700604.515161] LustreError: 68937:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST0009_UUID: namespace resource [0x540000400:0x6aef3:0x0].0x0 (ffff90afd8b243c0) refcount nonzero (2) after lock cleanup; forcing cleanup. [1700604.787129] LustreError: 137-5: fir-OST000b_UUID: not available for connect from 10.8.12.32@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700604.804622] LustreError: Skipped 2 previous similar messages [1700604.968329] Lustre: fir-OST0001: Not available for connect from 10.8.8.16@o2ib6 (stopping) [1700604.976774] Lustre: Skipped 66 previous similar messages [1700605.289964] Lustre: server umount fir-OST000b complete [1700605.301428] LustreError: 137-5: fir-OST000b_UUID: not available for connect from 10.8.26.27@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700605.318885] LustreError: Skipped 14 previous similar messages [1700605.710844] Lustre: Failing over fir-OST0003 [1700605.969241] Lustre: fir-OST0005: Not available for connect from 10.8.1.17@o2ib6 (stopping) [1700605.977678] Lustre: Skipped 92 previous similar messages [1700606.307897] LustreError: 137-5: fir-OST0003_UUID: not available for connect from 10.9.103.32@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700606.325439] LustreError: Skipped 26 previous similar messages [1700606.508920] Lustre: server umount fir-OST0003 complete [1700606.710871] Lustre: Failing over fir-OST0007 [1700608.310270] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.8.12.14@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700608.327727] LustreError: Skipped 155 previous similar messages [1700608.935434] Lustre: fir-OST0007: Not available for connect from 10.8.8.9@o2ib6 (stopping) [1700608.943790] Lustre: Skipped 40 previous similar messages [1700609.220376] md1: detected capacity change from 64011422924800 to 0 [1700609.225610] md3: detected capacity change from 64011422924800 to 0 [1700609.225619] md: md3 stopped. [1700609.236154] md: md1 stopped. [1700609.499095] md9: detected capacity change from 64011422924800 to 0 [1700609.505462] md: md9 stopped. [1700609.763624] Lustre: server umount fir-OST0007 complete [1700609.768940] Lustre: Skipped 3 previous similar messages [1700610.133064] md5: detected capacity change from 64011422924800 to 0 [1700610.139435] md: md5 stopped. [1700610.235949] md11: detected capacity change from 64011422924800 to 0 [1700610.242406] md: md11 stopped. [1700610.242602] md7: detected capacity change from 64011422924800 to 0 [1700610.242604] md: md7 stopped. [1700612.627525] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.210@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1700613.146434] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.210@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1700613.162089] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Skipped 12 previous similar messages [1700614.586351] LNet: Removed LNI 10.0.10.102@o2ib7 [1700640.972562] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1700640.980539] alg: No test for adler32 (adler32-zlib) [1700641.796727] Lustre: Lustre: Build Version: 2.12.0 [1700641.949529] LNet: Using FastReg for registration [1700641.966415] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [1700644.823756] md: md11 stopped. [1700644.877658] md/raid:md11: device dm-50 operational as raid disk 0 [1700644.883936] md/raid:md11: device dm-46 operational as raid disk 9 [1700644.890209] md/raid:md11: device dm-30 operational as raid disk 8 [1700644.896505] md/raid:md11: device dm-112 operational as raid disk 7 [1700644.902863] md/raid:md11: device dm-114 operational as raid disk 6 [1700644.909224] md/raid:md11: device dm-80 operational as raid disk 5 [1700644.915497] md/raid:md11: device dm-48 operational as raid disk 4 [1700644.921779] md/raid:md11: device dm-40 operational as raid disk 3 [1700644.928048] md/raid:md11: device dm-104 operational as raid disk 2 [1700644.934425] md/raid:md11: device dm-95 operational as raid disk 1 [1700644.941546] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700644.960479] md11: detected capacity change from 0 to 64011422924800 [1700644.968130] md: md1 stopped. [1700644.991886] md/raid:md1: device dm-26 operational as raid disk 0 [1700644.998094] md/raid:md1: device dm-9 operational as raid disk 9 [1700645.004206] md/raid:md1: device dm-16 operational as raid disk 8 [1700645.010481] md/raid:md1: device dm-59 operational as raid disk 7 [1700645.016675] md/raid:md1: device dm-23 operational as raid disk 6 [1700645.022862] md/raid:md1: device dm-60 operational as raid disk 5 [1700645.029067] md/raid:md1: device dm-34 operational as raid disk 4 [1700645.035315] md/raid:md1: device dm-68 operational as raid disk 3 [1700645.041501] md/raid:md1: device dm-43 operational as raid disk 2 [1700645.047689] md/raid:md1: device dm-57 operational as raid disk 1 [1700645.054678] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700645.073170] md1: detected capacity change from 0 to 64011422924800 [1700645.087357] md: md5 stopped. [1700645.112163] md/raid:md5: device dm-84 operational as raid disk 0 [1700645.118347] md/raid:md5: device dm-8 operational as raid disk 9 [1700645.124449] md/raid:md5: device dm-3 operational as raid disk 8 [1700645.130551] md/raid:md5: device dm-22 operational as raid disk 7 [1700645.136754] md/raid:md5: device dm-21 operational as raid disk 6 [1700645.142933] md/raid:md5: device dm-67 operational as raid disk 5 [1700645.149113] md/raid:md5: device dm-36 operational as raid disk 4 [1700645.155301] md/raid:md5: device dm-10 operational as raid disk 3 [1700645.161501] md/raid:md5: device dm-76 operational as raid disk 2 [1700645.167688] md/raid:md5: device dm-56 operational as raid disk 1 [1700645.174613] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700645.192227] md5: detected capacity change from 0 to 64011422924800 [1700645.207949] md: md9 stopped. [1700645.289037] md/raid:md9: device dm-47 operational as raid disk 0 [1700645.295240] md/raid:md9: device dm-42 operational as raid disk 9 [1700645.301438] md/raid:md9: device dm-39 operational as raid disk 8 [1700645.307636] md/raid:md9: device dm-28 operational as raid disk 7 [1700645.313834] md/raid:md9: device dm-116 operational as raid disk 6 [1700645.320128] md/raid:md9: device dm-71 operational as raid disk 5 [1700645.326323] md/raid:md9: device dm-110 operational as raid disk 4 [1700645.332607] md/raid:md9: device dm-94 operational as raid disk 3 [1700645.338810] md/raid:md9: device dm-33 operational as raid disk 2 [1700645.345008] md/raid:md9: device dm-53 operational as raid disk 1 [1700645.352373] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700645.371204] md9: detected capacity change from 0 to 64011422924800 [1700645.381414] md: md7 stopped. [1700645.411464] md/raid:md7: device dm-92 operational as raid disk 0 [1700645.417671] md/raid:md7: device dm-38 operational as raid disk 9 [1700645.423872] md/raid:md7: device dm-49 operational as raid disk 8 [1700645.430057] md/raid:md7: device dm-97 operational as raid disk 7 [1700645.436248] md/raid:md7: device dm-111 operational as raid disk 6 [1700645.442525] md/raid:md7: device dm-96 operational as raid disk 5 [1700645.448723] md/raid:md7: device dm-101 operational as raid disk 4 [1700645.454997] md/raid:md7: device dm-91 operational as raid disk 3 [1700645.461185] md/raid:md7: device dm-86 operational as raid disk 2 [1700645.467373] md/raid:md7: device dm-109 operational as raid disk 1 [1700645.474396] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700645.493409] md7: detected capacity change from 0 to 64011422924800 [1700645.511422] md: md3 stopped. [1700645.545315] md/raid:md3: device dm-2 operational as raid disk 0 [1700645.551426] md/raid:md3: device dm-0 operational as raid disk 9 [1700645.557539] md/raid:md3: device dm-65 operational as raid disk 8 [1700645.563734] md/raid:md3: device dm-24 operational as raid disk 7 [1700645.569927] md/raid:md3: device dm-14 operational as raid disk 6 [1700645.576124] md/raid:md3: device dm-118 operational as raid disk 5 [1700645.582404] md/raid:md3: device dm-64 operational as raid disk 4 [1700645.588598] md/raid:md3: device dm-75 operational as raid disk 3 [1700645.594795] md/raid:md3: device dm-31 operational as raid disk 2 [1700645.600988] md/raid:md3: device dm-15 operational as raid disk 1 [1700645.608229] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700645.635331] md3: detected capacity change from 0 to 64011422924800 [1700645.971317] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [1700646.080348] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1700646.367336] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [1700646.383516] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700646.423051] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700646.439331] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1700646.751832] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700646.809366] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [1700646.900771] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [1700646.925386] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700647.171590] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700647.228488] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700651.949315] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700652.259795] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.8.23.21@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700652.277256] LustreError: Skipped 2 previous similar messages [1700654.791043] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.9.103.25@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700654.808598] LustreError: Skipped 5 previous similar messages [1700655.809111] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.12.8@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700655.826486] LustreError: Skipped 64 previous similar messages [1700657.838116] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.8.15.1@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700657.855479] LustreError: Skipped 137 previous similar messages [1700657.949478] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700659.949523] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1700661.895884] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.0.65@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700661.913265] LustreError: Skipped 319 previous similar messages [1700661.949576] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 4 seconds [1700661.959831] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1700661.971510] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [1700661.980477] Lustre: fir-OST000b: in recovery but waiting for the first client to connect [1700667.829397] Lustre: fir-OST000b: Will be in recovery for at least 5:00, or until 435 clients reconnect [1700667.838919] Lustre: fir-OST000b: Connection restored to (at 10.9.101.28@o2ib4) [1700667.949734] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700668.353815] Lustre: fir-OST000b: Connection restored to (at 10.8.18.9@o2ib6) [1700668.361138] Lustre: Skipped 5 previous similar messages [1700669.405450] Lustre: fir-OST000b: Connection restored to (at 10.9.105.11@o2ib4) [1700669.412937] Lustre: Skipped 33 previous similar messages [1700669.937692] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.9.101.71@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700669.955239] LustreError: Skipped 1326 previous similar messages [1700671.457526] Lustre: fir-OST000b: Connection restored to af82ddae-4046-4f89-45bb-baefabf8e06e (at 10.9.114.1@o2ib4) [1700671.468066] Lustre: Skipped 58 previous similar messages [1700671.952914] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [1700671.961851] Lustre: fir-OST0001: in recovery but waiting for the first client to connect [1700672.010231] Lustre: fir-OST0001: Will be in recovery for at least 5:00, or until 398 clients reconnect [1700672.218867] Lustre: fir-OST0005: Not available for connect from 10.8.27.15@o2ib6 (not set up) [1700675.688607] Lustre: fir-OST000b: Connection restored to (at 10.8.22.20@o2ib6) [1700675.696015] Lustre: Skipped 122 previous similar messages [1700675.949935] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 8 seconds [1700675.960189] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [1700681.954188] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [1700681.963210] Lustre: fir-OST0005: in recovery but waiting for the first client to connect [1700682.060185] Lustre: fir-OST0005: Will be in recovery for at least 5:00, or until 409 clients reconnect [1700683.698154] Lustre: fir-OST000b: Connection restored to (at 10.8.10.6@o2ib6) [1700683.705473] Lustre: Skipped 116 previous similar messages [1700685.953873] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.8.2.18@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700685.971245] LustreError: Skipped 642 previous similar messages [1700686.953654] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [1700686.962591] Lustre: fir-OST0009: in recovery but waiting for the first client to connect [1700686.981113] Lustre: fir-OST0009: Will be in recovery for at least 5:00, or until 397 clients reconnect [1700687.221416] Lustre: fir-OST0003: Not available for connect from 10.8.7.11@o2ib6 (not set up) [1700687.230023] Lustre: Skipped 1 previous similar message [1700691.950345] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 10 seconds [1700691.960688] LNet: 69772:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 8 previous similar messages [1700691.972283] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [1700691.981220] Lustre: fir-OST0003: in recovery but waiting for the first client to connect [1700691.999535] Lustre: fir-OST0003: Will be in recovery for at least 5:00, or until 372 clients reconnect [1700692.298211] Lustre: fir-OST0007: Not available for connect from 10.8.12.30@o2ib6 (not set up) [1700692.306916] Lustre: Skipped 1 previous similar message [1700699.839624] Lustre: fir-OST0005: Connection restored to (at 10.9.102.14@o2ib4) [1700699.847117] Lustre: Skipped 981 previous similar messages [1700700.452657] Lustre: fir-OST0001: Denying connection for new client 422c37d0-97a3-ea50-ac82-e8f84679c682(at 10.8.26.2@o2ib6), waiting for 398 known clients (233 recovered, 162 in progress, and 0 evicted) already passed deadline 0:28 [1700700.952905] Lustre: fir-OST0007: Imperative Recovery not enabled, recovery window 300-900 [1700700.965074] Lustre: fir-OST0007: in recovery but waiting for the first client to connect [1700702.832002] Lustre: fir-OST0007: Will be in recovery for at least 5:00, or until 418 clients reconnect [1700703.026225] Lustre: fir-OST000b: Recovery over after 0:36, of 435 clients 435 recovered and 0 were evicted. [1700706.182425] Lustre: fir-OST0003: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 372 known clients (51 recovered, 35 in progress, and 0 evicted) already passed deadline 0:14 [1700706.203001] Lustre: Skipped 1 previous similar message [1700708.480612] Lustre: fir-OST0009: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 397 known clients (99 recovered, 64 in progress, and 0 evicted) already passed deadline 0:22 [1700708.500061] Lustre: Skipped 2 previous similar messages [1700714.749601] Lustre: fir-OST0001: Denying connection for new client 1e03043d-571f-f709-9076-72fb7d056ac3(at 10.9.103.43@o2ib4), waiting for 398 known clients (234 recovered, 163 in progress, and 0 evicted) already passed deadline 0:42 [1700714.770439] Lustre: Skipped 7 previous similar messages [1700725.541310] Lustre: fir-OST0003: Denying connection for new client 422c37d0-97a3-ea50-ac82-e8f84679c682(at 10.8.26.2@o2ib6), waiting for 372 known clients (184 recovered, 104 in progress, and 0 evicted) already passed deadline 0:34 [1700725.561976] Lustre: Skipped 2 previous similar messages [1700731.848959] Lustre: fir-OST0009: Connection restored to (at 10.9.106.56@o2ib4) [1700731.856452] Lustre: Skipped 855 previous similar messages [1700732.270234] Lustre: fir-OST0009: Recovery over after 0:46, of 397 clients 397 recovered and 0 were evicted. [1700733.563790] Lustre: fir-OST0005: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 409 known clients (257 recovered, 151 in progress, and 0 evicted) already passed deadline 0:51 [1700733.583428] Lustre: Skipped 5 previous similar messages [1700750.629867] Lustre: fir-OST0001: Denying connection for new client 422c37d0-97a3-ea50-ac82-e8f84679c682(at 10.8.26.2@o2ib6), waiting for 398 known clients (234 recovered, 163 in progress, and 0 evicted) already passed deadline 1:18 [1700750.650531] Lustre: Skipped 12 previous similar messages [1700783.724213] Lustre: fir-OST0005: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 409 known clients (257 recovered, 151 in progress, and 0 evicted) already passed deadline 1:41 [1700783.743837] Lustre: Skipped 24 previous similar messages [1700809.267345] Lustre: fir-OST000b: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [1700809.276733] Lustre: Skipped 256 previous similar messages [1700850.984347] Lustre: fir-OST0001: Denying connection for new client 422c37d0-97a3-ea50-ac82-e8f84679c682(at 10.8.26.2@o2ib6), waiting for 398 known clients (234 recovered, 163 in progress, and 0 evicted) already passed deadline 2:58 [1700851.005028] Lustre: Skipped 75 previous similar messages [1700963.439851] Lustre: fir-OST000b: Connection restored to 9a86fbfd-6aa1-17d8-3e18-f77ffac8735a (at 10.8.21.25@o2ib6) [1700963.450381] Lustre: Skipped 27 previous similar messages [1700972.854523] Lustre: fir-OST0001: recovery is timed out, evict stale exports [1700972.861739] Lustre: fir-OST0001: disconnecting 1 stale clients [1700973.000818] Lustre: fir-OST0001: Recovery over after 5:01, of 398 clients 397 recovered and 1 was evicted. [1700982.157281] Lustre: fir-OST0003: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 372 known clients (246 recovered, 125 in progress, and 0 evicted) already passed deadline 4:50 [1700982.178031] Lustre: Skipped 143 previous similar messages [1700983.094788] Lustre: fir-OST0005: recovery is timed out, evict stale exports [1700983.102013] Lustre: fir-OST0005: disconnecting 1 stale clients [1700983.269266] Lustre: fir-OST0005: Recovery over after 5:01, of 409 clients 408 recovered and 1 was evicted. [1700992.311020] Lustre: fir-OST0003: recovery is timed out, evict stale exports [1700992.318242] Lustre: fir-OST0003: disconnecting 1 stale clients [1700992.416850] Lustre: fir-OST0003: Recovery over after 5:01, of 372 clients 371 recovered and 1 was evicted. [1701003.575305] Lustre: fir-OST0007: recovery is timed out, evict stale exports [1701003.582510] Lustre: fir-OST0007: disconnecting 1 stale clients [1701003.713466] Lustre: fir-OST0007: Recovery over after 5:01, of 418 clients 417 recovered and 1 was evicted. [1701202.620564] Lustre: fir-MDT0003-lwp-OST000b: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1701202.636733] Lustre: Skipped 8 previous similar messages [1701221.940895] Lustre: 79143:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549478931/real 1549478931] req@ffff90ba09e50300 x1624745817680512/t0(0) o39->fir-MDT0002-lwp-OST0007@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478937 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1701221.969376] Lustre: 79143:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1701227.942055] Lustre: 79141:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549478937/real 1549478937] req@ffff90b89a9d8000 x1624745817680608/t0(0) o39->fir-MDT0000-lwp-OST000b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478943 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1701227.947566] Lustre: Failing over fir-OST0005 [1701227.968464] LustreError: 79140:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST0005_UUID: namespace resource [0xa5920:0x0:0x0].0x0 (ffff90b4cfd59080) refcount nonzero (2) after lock cleanup; forcing cleanup. [1701227.994933] Lustre: 79141:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1701228.723143] Lustre: server umount fir-OST0001 complete [1701228.728461] Lustre: Skipped 1 previous similar message [1701228.966944] Lustre: fir-OST0003: Not available for connect from 10.9.101.52@o2ib4 (stopping) [1701228.994821] md9: detected capacity change from 64011422924800 to 0 [1701229.001194] md: md9 stopped. [1701229.037534] md1: detected capacity change from 64011422924800 to 0 [1701229.043908] md: md1 stopped. [1701229.298552] LustreError: 137-5: fir-OST0009_UUID: not available for connect from 10.8.23.21@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701229.316017] LustreError: Skipped 754 previous similar messages [1701230.796711] Lustre: server umount fir-OST0003 complete [1701230.802027] Lustre: Skipped 1 previous similar message [1701231.114019] md11: detected capacity change from 64011422924800 to 0 [1701231.120476] md: md11 stopped. [1701231.163404] md7: detected capacity change from 64011422924800 to 0 [1701231.165297] md5: detected capacity change from 64011422924800 to 0 [1701231.165306] md: md5 stopped. [1701231.179182] md: md7 stopped. [1701231.179382] md3: detected capacity change from 64011422924800 to 0 [1701231.179385] md: md3 stopped. [1701233.016069] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.203@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1701233.811396] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.201@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1701233.827050] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Skipped 14 previous similar messages [1701235.001226] LNet: Removed LNI 10.0.10.102@o2ib7 [1701287.525269] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1701287.532984] alg: No test for adler32 (adler32-zlib) [1701288.338645] Lustre: Lustre: Build Version: 2.12.0 [1701288.465791] LNet: Using FastReg for registration [1701288.483308] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [1701293.913572] md: md9 stopped. [1701293.958068] md/raid:md9: device dm-47 operational as raid disk 0 [1701293.964255] md/raid:md9: device dm-42 operational as raid disk 9 [1701293.970447] md/raid:md9: device dm-39 operational as raid disk 8 [1701293.976638] md/raid:md9: device dm-28 operational as raid disk 7 [1701293.982831] md/raid:md9: device dm-116 operational as raid disk 6 [1701293.989107] md/raid:md9: device dm-71 operational as raid disk 5 [1701293.995305] md/raid:md9: device dm-110 operational as raid disk 4 [1701294.001588] md/raid:md9: device dm-94 operational as raid disk 3 [1701294.007782] md/raid:md9: device dm-33 operational as raid disk 2 [1701294.013972] md/raid:md9: device dm-53 operational as raid disk 1 [1701294.021080] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701294.038509] md9: detected capacity change from 0 to 64011422924800 [1701294.046125] md: md7 stopped. [1701294.080625] md/raid:md7: device dm-92 operational as raid disk 0 [1701294.086815] md/raid:md7: device dm-38 operational as raid disk 9 [1701294.093007] md/raid:md7: device dm-49 operational as raid disk 8 [1701294.099204] md/raid:md7: device dm-97 operational as raid disk 7 [1701294.105399] md/raid:md7: device dm-111 operational as raid disk 6 [1701294.111675] md/raid:md7: device dm-96 operational as raid disk 5 [1701294.117872] md/raid:md7: device dm-101 operational as raid disk 4 [1701294.124156] md/raid:md7: device dm-91 operational as raid disk 3 [1701294.130351] md/raid:md7: device dm-86 operational as raid disk 2 [1701294.136540] md/raid:md7: device dm-109 operational as raid disk 1 [1701294.143600] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701294.168541] md7: detected capacity change from 0 to 64011422924800 [1701294.182582] md: md1 stopped. [1701294.229219] md/raid:md1: device dm-26 operational as raid disk 0 [1701294.235415] md/raid:md1: device dm-9 operational as raid disk 9 [1701294.241523] md/raid:md1: device dm-16 operational as raid disk 8 [1701294.247715] md/raid:md1: device dm-59 operational as raid disk 7 [1701294.253923] md/raid:md1: device dm-23 operational as raid disk 6 [1701294.260128] md/raid:md1: device dm-60 operational as raid disk 5 [1701294.266339] md/raid:md1: device dm-34 operational as raid disk 4 [1701294.272526] md/raid:md1: device dm-68 operational as raid disk 3 [1701294.278713] md/raid:md1: device dm-43 operational as raid disk 2 [1701294.284892] md/raid:md1: device dm-57 operational as raid disk 1 [1701294.292139] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701294.310437] md1: detected capacity change from 0 to 64011422924800 [1701294.324319] md: md11 stopped. [1701294.357109] md/raid:md11: device dm-50 operational as raid disk 0 [1701294.363387] md/raid:md11: device dm-46 operational as raid disk 9 [1701294.369710] md/raid:md11: device dm-30 operational as raid disk 8 [1701294.375994] md/raid:md11: device dm-112 operational as raid disk 7 [1701294.382405] md/raid:md11: device dm-114 operational as raid disk 6 [1701294.388847] md/raid:md11: device dm-80 operational as raid disk 5 [1701294.395168] md/raid:md11: device dm-48 operational as raid disk 4 [1701294.401447] md/raid:md11: device dm-40 operational as raid disk 3 [1701294.407760] md/raid:md11: device dm-104 operational as raid disk 2 [1701294.414148] md/raid:md11: device dm-95 operational as raid disk 1 [1701294.421240] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701294.438268] md11: detected capacity change from 0 to 64011422924800 [1701294.449935] md: md5 stopped. [1701294.483381] md/raid:md5: device dm-84 operational as raid disk 0 [1701294.489582] md/raid:md5: device dm-8 operational as raid disk 9 [1701294.495695] md/raid:md5: device dm-3 operational as raid disk 8 [1701294.501823] md/raid:md5: device dm-22 operational as raid disk 7 [1701294.508016] md/raid:md5: device dm-21 operational as raid disk 6 [1701294.514213] md/raid:md5: device dm-67 operational as raid disk 5 [1701294.520415] md/raid:md5: device dm-36 operational as raid disk 4 [1701294.526604] md/raid:md5: device dm-10 operational as raid disk 3 [1701294.532797] md/raid:md5: device dm-76 operational as raid disk 2 [1701294.538982] md/raid:md5: device dm-56 operational as raid disk 1 [1701294.546200] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701294.569435] md5: detected capacity change from 0 to 64011422924800 [1701294.603580] md: md3 stopped. [1701294.665955] md/raid:md3: device dm-2 operational as raid disk 0 [1701294.672067] md/raid:md3: device dm-0 operational as raid disk 9 [1701294.678212] md/raid:md3: device dm-65 operational as raid disk 8 [1701294.684474] md/raid:md3: device dm-24 operational as raid disk 7 [1701294.690706] md/raid:md3: device dm-14 operational as raid disk 6 [1701294.696899] md/raid:md3: device dm-118 operational as raid disk 5 [1701294.703177] md/raid:md3: device dm-64 operational as raid disk 4 [1701294.709376] md/raid:md3: device dm-75 operational as raid disk 3 [1701294.715568] md/raid:md3: device dm-31 operational as raid disk 2 [1701294.721769] md/raid:md3: device dm-15 operational as raid disk 1 [1701294.729730] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701294.759917] md3: detected capacity change from 0 to 64011422924800 [1701294.905929] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1701295.012921] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [1701295.204270] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701295.320799] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701295.480982] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1701295.717984] LDISKFS-fs (md5): file extents enabled [1701295.719964] LDISKFS-fs (md11): file extents enabled [1701295.719965] , maximum tree depth=5 [1701295.729774] , maximum tree depth=5 [1701295.776483] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701296.078191] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [1701296.145825] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701296.154279] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701296.395739] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701296.464837] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1701296.676334] LustreError: 137-5: fir-OST0001_UUID: not available for connect from 10.9.101.24@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701298.214861] LustreError: 137-5: fir-OST000b_UUID: not available for connect from 10.9.101.34@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701298.232403] LustreError: Skipped 1 previous similar message [1701299.762349] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.9.103.35@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701301.806641] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.8.23.3@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701301.824015] LustreError: Skipped 39 previous similar messages [1701302.464962] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1701304.465017] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1701304.475276] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1701306.026007] LustreError: 137-5: fir-OST000b_UUID: not available for connect from 10.8.11.18@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701306.043489] LustreError: Skipped 115 previous similar messages [1701306.468034] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [1701306.477017] Lustre: fir-OST0009: in recovery but waiting for the first client to connect [1701306.607871] Lustre: fir-OST0009: Will be in recovery for at least 5:00, or until 192 clients reconnect [1701306.617419] Lustre: fir-OST0009: Connection restored to (at 10.8.21.15@o2ib6) [1701307.241103] Lustre: fir-OST0009: Connection restored to (at 10.8.2.23@o2ib6) [1701307.248419] Lustre: Skipped 3 previous similar messages [1701307.465099] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 5 seconds [1701307.475356] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1701308.254443] Lustre: fir-OST0009: Connection restored to (at 10.8.1.21@o2ib6) [1701308.261769] Lustre: Skipped 6 previous similar messages [1701310.256128] Lustre: fir-OST0009: Connection restored to (at 10.8.17.15@o2ib6) [1701310.263533] Lustre: Skipped 30 previous similar messages [1701311.467561] Lustre: fir-OST0007: Imperative Recovery not enabled, recovery window 300-900 [1701311.478705] Lustre: fir-OST0007: in recovery but waiting for the first client to connect [1701311.501951] Lustre: fir-OST0007: Will be in recovery for at least 5:00, or until 203 clients reconnect [1701311.566318] Lustre: fir-OST0001: Not available for connect from 10.8.24.16@o2ib6 (not set up) [1701312.465222] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 10 seconds [1701312.475568] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1701314.032436] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.8.13.15@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701314.049899] LustreError: Skipped 343 previous similar messages [1701314.379074] Lustre: fir-OST0009: Connection restored to (at 10.8.23.27@o2ib6) [1701314.386476] Lustre: Skipped 126 previous similar messages [1701321.465457] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [1701321.475718] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1701321.487443] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [1701321.496555] Lustre: fir-OST0001: in recovery but waiting for the first client to connect [1701323.269070] Lustre: fir-OST0009: Connection restored to (at 10.9.104.71@o2ib4) [1701323.276562] Lustre: Skipped 107 previous similar messages [1701325.442310] Lustre: fir-OST0001: Will be in recovery for at least 5:00, or until 200 clients reconnect [1701330.065414] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.8.26.8@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701330.082810] LustreError: Skipped 354 previous similar messages [1701331.468007] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [1701331.478693] Lustre: fir-OST000b: in recovery but waiting for the first client to connect [1701336.468672] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [1701336.480690] Lustre: fir-OST0005: in recovery but waiting for the first client to connect [1701336.693092] Lustre: fir-OST000b: Will be in recovery for at least 5:00, or until 232 clients reconnect [1701336.851458] Lustre: fir-OST0009: Recovery over after 0:30, of 192 clients 192 recovered and 0 were evicted. [1701338.465892] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 11 seconds [1701338.476239] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 9 previous similar messages [1701339.287258] Lustre: fir-OST0001: Connection restored to 11bdb068-3a68-b486-d427-87b2a02899d5 (at 10.8.2.24@o2ib6) [1701339.297696] Lustre: Skipped 131 previous similar messages [1701349.940033] Lustre: fir-OST0005: Will be in recovery for at least 5:00, or until 200 clients reconnect [1701362.247177] Lustre: fir-OST0007: Recovery over after 0:51, of 203 clients 203 recovered and 0 were evicted. [1701363.935185] Lustre: fir-OST000b: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 232 known clients (66 recovered, 69 in progress, and 0 evicted) already passed deadline 0:26 [1701363.954639] Lustre: Skipped 1 previous similar message [1701364.994748] Lustre: fir-OST000b: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 232 known clients (76 recovered, 86 in progress, and 0 evicted) already passed deadline 0:27 [1701365.014197] Lustre: Skipped 1 previous similar message [1701371.446922] Lustre: fir-OST0005: Connection restored to 028d6433-9e7d-1b84-c8b7-1bb2a8570ec4 (at 10.8.1.4@o2ib6) [1701371.457270] Lustre: Skipped 783 previous similar messages [1701371.723063] Lustre: fir-OST000b: Recovery over after 0:35, of 232 clients 232 recovered and 0 were evicted. [1701371.732979] Lustre: Skipped 1 previous similar message [1701375.756351] Lustre: fir-OST0005: Recovery over after 0:26, of 200 clients 200 recovered and 0 were evicted. [1701386.697951] Lustre: fir-OST0003: Recovery over after 0:36, of 174 clients 174 recovered and 0 were evicted. [1701468.241484] Lustre: fir-OST000b: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [1701468.250885] Lustre: Skipped 17 previous similar messages [1702190.309687] Lustre: 93727:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549479900/real 1549479900] req@ffff90a9a86b0000 x1624746496114224/t0(0) o39->fir-MDT0002-lwp-OST0003@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479906 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1702190.338150] Lustre: 93727:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1702194.487793] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [1702194.498138] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1702194.507658] Lustre: 80484:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549479910/real 1549479910] req@ffff90d841959500 x1624746496114544/t0(0) o400->fir-MDT0003-lwp-OST0005@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549480666 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1702194.507672] Lustre: fir-MDT0001-lwp-OST0005: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1702194.552744] Lustre: 80484:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1702196.310842] Lustre: 93726:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549479906/real 1549479906] req@ffff90ac8d248000 x1624746496114288/t0(0) o39->fir-MDT0000-lwp-OST0007@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479912 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1702196.339303] Lustre: 93726:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1702196.487893] Lustre: fir-MDT0003-lwp-OST0003: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1702196.504048] Lustre: Skipped 5 previous similar messages [1702197.489747] Lustre: Failing over fir-OST0005 [1702197.494202] Lustre: Skipped 2 previous similar messages [1702197.507151] LustreError: 93725:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST0001_UUID: namespace resource [0xa58c0:0x0:0x0].0x0 (ffff90caca19db00) refcount nonzero (2) after lock cleanup; forcing cleanup. [1702197.527169] LustreError: 93725:0:(ldlm_resource.c:1146:ldlm_resource_complain()) Skipped 57 previous similar messages [1702197.550804] Lustre: fir-OST0005: Not available for connect from 10.8.2.17@o2ib6 (stopping) [1702198.487961] Lustre: 93724:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549479913/real 1549479914] req@ffff90bb5f6a8000 x1624746496114752/t0(0) o39->fir-MDT0003-lwp-OST000b@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479919 ref 2 fl Rpc:eXN/0/ffffffff rc 0/-1 [1702198.491670] Lustre: Failing over fir-OST0003 [1702198.510228] LustreError: 93727:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST0003_UUID: namespace resource [0xbc0000400:0x23a00:0x0].0x0 (ffff90dbf6aafc80) refcount nonzero (2) after lock cleanup; forcing cleanup. [1702198.510230] LustreError: 93727:0:(ldlm_resource.c:1146:ldlm_resource_complain()) Skipped 72 previous similar messages [1702198.552712] Lustre: 93724:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1702198.871573] Lustre: fir-OST0003: Not available for connect from 10.8.6.13@o2ib6 (stopping) [1702198.880016] Lustre: Skipped 2 previous similar messages [1702199.487943] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1702199.498205] LNet: 80404:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 19 previous similar messages [1702199.507824] Lustre: fir-MDT0003-lwp-OST0009: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1702199.509651] Lustre: Failing over fir-OST000b [1702199.526148] LustreError: 93724:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST000b_UUID: namespace resource [0x380000401:0x2347f:0x0].0x0 (ffff90dbf8604180) refcount nonzero (2) after lock cleanup; forcing cleanup. [1702199.526150] LustreError: 93724:0:(ldlm_resource.c:1146:ldlm_resource_complain()) Skipped 78 previous similar messages [1702199.923893] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.8.6.11@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702199.941262] LustreError: Skipped 1008 previous similar messages [1702200.338210] Lustre: server umount fir-OST0007 complete [1702200.605975] md5: detected capacity change from 64011422924800 to 0 [1702200.612348] md: md5 stopped. [1702200.642968] md1: detected capacity change from 64011422924800 to 0 [1702200.649342] md: md1 stopped. [1702200.676535] md7: detected capacity change from 64011422924800 to 0 [1702200.682907] md: md7 stopped. [1702201.369185] Lustre: server umount fir-OST0003 complete [1702201.372030] Lustre: fir-OST000b: Not available for connect from 10.9.104.71@o2ib4 (stopping) [1702201.372031] Lustre: Skipped 11 previous similar messages [1702201.388597] Lustre: Skipped 2 previous similar messages [1702201.686281] md3: detected capacity change from 64011422924800 to 0 [1702201.692649] md: md3 stopped. [1702202.371908] Lustre: server umount fir-OST000b complete [1702202.684370] md9: detected capacity change from 64011422924800 to 0 [1702202.688873] md11: detected capacity change from 64011422924800 to 0 [1702202.688881] md: md11 stopped. [1702202.700322] md: md9 stopped. [1702204.649098] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.202@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1702205.203979] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.203@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1702205.219626] LNetError: 80169:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Skipped 10 previous similar messages [1702206.521094] LNet: Removed LNI 10.0.10.102@o2ib7 [1702253.736161] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1702253.744062] alg: No test for adler32 (adler32-zlib) [1702254.548069] Lustre: Lustre: Build Version: 2.12.0 [1702254.674271] LNet: Using FastReg for registration [1702254.691674] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [1702257.543025] md: md11 stopped. [1702257.580723] md/raid:md11: device dm-50 operational as raid disk 0 [1702257.586996] md/raid:md11: device dm-46 operational as raid disk 9 [1702257.593276] md/raid:md11: device dm-30 operational as raid disk 8 [1702257.599556] md/raid:md11: device dm-112 operational as raid disk 7 [1702257.605917] md/raid:md11: device dm-114 operational as raid disk 6 [1702257.612279] md/raid:md11: device dm-80 operational as raid disk 5 [1702257.618552] md/raid:md11: device dm-48 operational as raid disk 4 [1702257.624829] md/raid:md11: device dm-40 operational as raid disk 3 [1702257.631105] md/raid:md11: device dm-104 operational as raid disk 2 [1702257.637465] md/raid:md11: device dm-95 operational as raid disk 1 [1702257.644578] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702257.662497] md11: detected capacity change from 0 to 64011422924800 [1702257.677434] md: md5 stopped. [1702257.720204] md/raid:md5: device dm-84 operational as raid disk 0 [1702257.726406] md/raid:md5: device dm-8 operational as raid disk 9 [1702257.732506] md/raid:md5: device dm-3 operational as raid disk 8 [1702257.738606] md/raid:md5: device dm-22 operational as raid disk 7 [1702257.744796] md/raid:md5: device dm-21 operational as raid disk 6 [1702257.750983] md/raid:md5: device dm-67 operational as raid disk 5 [1702257.757170] md/raid:md5: device dm-36 operational as raid disk 4 [1702257.763359] md/raid:md5: device dm-10 operational as raid disk 3 [1702257.769546] md/raid:md5: device dm-76 operational as raid disk 2 [1702257.775735] md/raid:md5: device dm-56 operational as raid disk 1 [1702257.782720] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702257.807749] md5: detected capacity change from 0 to 64011422924800 [1702257.829582] md: md1 stopped. [1702257.877339] md/raid:md1: device dm-26 operational as raid disk 0 [1702257.883533] md/raid:md1: device dm-9 operational as raid disk 9 [1702257.889635] md/raid:md1: device dm-16 operational as raid disk 8 [1702257.895825] md/raid:md1: device dm-59 operational as raid disk 7 [1702257.902015] md/raid:md1: device dm-23 operational as raid disk 6 [1702257.908199] md/raid:md1: device dm-60 operational as raid disk 5 [1702257.914391] md/raid:md1: device dm-34 operational as raid disk 4 [1702257.920578] md/raid:md1: device dm-68 operational as raid disk 3 [1702257.926766] md/raid:md1: device dm-43 operational as raid disk 2 [1702257.932953] md/raid:md1: device dm-57 operational as raid disk 1 [1702257.940180] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702257.979373] md1: detected capacity change from 0 to 64011422924800 [1702257.992656] md: md3 stopped. [1702258.038903] md/raid:md3: device dm-2 operational as raid disk 0 [1702258.045015] md/raid:md3: device dm-0 operational as raid disk 9 [1702258.051121] md/raid:md3: device dm-65 operational as raid disk 8 [1702258.057313] md/raid:md3: device dm-24 operational as raid disk 7 [1702258.063511] md/raid:md3: device dm-14 operational as raid disk 6 [1702258.069728] md/raid:md3: device dm-118 operational as raid disk 5 [1702258.076005] md/raid:md3: device dm-64 operational as raid disk 4 [1702258.082196] md/raid:md3: device dm-75 operational as raid disk 3 [1702258.088385] md/raid:md3: device dm-31 operational as raid disk 2 [1702258.094574] md/raid:md3: device dm-15 operational as raid disk 1 [1702258.101602] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702258.125555] md3: detected capacity change from 0 to 64011422924800 [1702258.159014] md: md9 stopped. [1702258.212875] md/raid:md9: device dm-47 operational as raid disk 0 [1702258.219072] md/raid:md9: device dm-42 operational as raid disk 9 [1702258.225265] md/raid:md9: device dm-39 operational as raid disk 8 [1702258.231454] md/raid:md9: device dm-28 operational as raid disk 7 [1702258.237641] md/raid:md9: device dm-116 operational as raid disk 6 [1702258.243917] md/raid:md9: device dm-71 operational as raid disk 5 [1702258.250107] md/raid:md9: device dm-110 operational as raid disk 4 [1702258.256390] md/raid:md9: device dm-94 operational as raid disk 3 [1702258.262589] md/raid:md9: device dm-33 operational as raid disk 2 [1702258.268782] md/raid:md9: device dm-53 operational as raid disk 1 [1702258.276214] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702258.298751] md9: detected capacity change from 0 to 64011422924800 [1702258.318020] md: md7 stopped. [1702258.356800] md/raid:md7: device dm-92 operational as raid disk 0 [1702258.362990] md/raid:md7: device dm-38 operational as raid disk 9 [1702258.369178] md/raid:md7: device dm-49 operational as raid disk 8 [1702258.375384] md/raid:md7: device dm-97 operational as raid disk 7 [1702258.381569] md/raid:md7: device dm-111 operational as raid disk 6 [1702258.386625] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [1702258.394853] md/raid:md7: device dm-96 operational as raid disk 5 [1702258.401072] md/raid:md7: device dm-101 operational as raid disk 4 [1702258.407359] md/raid:md7: device dm-91 operational as raid disk 3 [1702258.413554] md/raid:md7: device dm-86 operational as raid disk 2 [1702258.419765] md/raid:md7: device dm-109 operational as raid disk 1 [1702258.426760] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702258.445792] md7: detected capacity change from 0 to 64011422924800 [1702258.634618] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [1702258.677873] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702258.931254] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702259.010624] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1702259.116629] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [1702259.151644] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1702259.385606] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702259.429613] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [1702259.449726] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702259.499645] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702259.749922] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702262.673540] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1702263.727786] LustreError: 137-5: fir-OST0003_UUID: not available for connect from 10.8.27.19@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702263.745251] LustreError: Skipped 3 previous similar messages [1702264.732720] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.8.6.11@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702264.750087] LustreError: Skipped 6 previous similar messages [1702267.256873] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.9.107.6@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702267.274334] LustreError: Skipped 3 previous similar messages [1702269.354664] LustreError: 137-5: fir-OST0001_UUID: not available for connect from 10.8.12.8@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702269.372034] LustreError: Skipped 25 previous similar messages [1702269.673729] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1702272.673794] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [1702272.684047] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1702273.447978] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.8.20.17@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702273.465440] LustreError: Skipped 45 previous similar messages [1702273.676003] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [1702273.685016] Lustre: fir-OST000b: in recovery but waiting for the first client to connect [1702273.688512] Lustre: fir-OST000b: Will be in recovery for at least 5:00, or until 126 clients reconnect [1702273.688787] Lustre: fir-OST000b: Connection restored to (at 10.8.22.31@o2ib6) [1702274.275858] Lustre: fir-OST000b: Connection restored to cc107cab-3544-aeaa-6b27-e00a056fcf80 (at 10.8.1.26@o2ib6) [1702274.286295] Lustre: Skipped 6 previous similar messages [1702275.324610] Lustre: fir-OST000b: Connection restored to cd4830da-6dd3-dc14-e86c-794e3eb4bb0b (at 10.8.8.24@o2ib6) [1702275.335046] Lustre: Skipped 15 previous similar messages [1702275.673885] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 6 seconds [1702275.684139] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1702277.339705] Lustre: fir-OST000b: Connection restored to (at 10.8.23.35@o2ib6) [1702277.347110] Lustre: Skipped 32 previous similar messages [1702278.676745] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [1702278.685692] Lustre: fir-OST0005: in recovery but waiting for the first client to connect [1702278.740001] Lustre: fir-OST0001: Not available for connect from 10.8.30.33@o2ib6 (not set up) [1702278.748493] Lustre: fir-OST0005: Will be in recovery for at least 5:00, or until 106 clients reconnect [1702280.674005] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 11 seconds [1702280.684345] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1702281.387633] Lustre: fir-OST000b: Connection restored to (at 10.8.10.30@o2ib6) [1702281.395048] Lustre: Skipped 85 previous similar messages [1702281.457191] LustreError: 137-5: fir-OST0003_UUID: not available for connect from 10.8.7.10@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702281.474553] LustreError: Skipped 348 previous similar messages [1702283.676326] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [1702283.685344] Lustre: fir-OST0001: in recovery but waiting for the first client to connect [1702283.705363] Lustre: fir-OST0001: Will be in recovery for at least 5:00, or until 107 clients reconnect [1702288.676492] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [1702288.688617] Lustre: fir-OST0003: in recovery but waiting for the first client to connect [1702289.674253] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [1702289.684594] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [1702290.112664] Lustre: fir-OST000b: Connection restored to (at 10.9.101.34@o2ib4) [1702290.120153] Lustre: Skipped 61 previous similar messages [1702293.677176] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [1702293.686198] Lustre: fir-OST0009: in recovery but waiting for the first client to connect [1702297.543474] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.8.11.33@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702297.560932] LustreError: Skipped 77 previous similar messages [1702298.786578] Lustre: fir-OST000b: Recovery over after 0:25, of 126 clients 126 recovered and 0 were evicted. [1702304.918546] Lustre: fir-OST0007: Will be in recovery for at least 5:00, or until 95 clients reconnect [1702306.266161] Lustre: fir-OST0003: Connection restored to (at 10.8.6.13@o2ib6) [1702306.273475] Lustre: Skipped 59 previous similar messages [1702313.882333] Lustre: fir-OST0009: Will be in recovery for at least 5:00, or until 99 clients reconnect [1702313.891748] Lustre: Skipped 1 previous similar message [1702318.624844] Lustre: fir-OST0009: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 99 known clients (1 recovered, 4 in progress, and 0 evicted) already passed deadline 0:05 [1702318.644038] Lustre: Skipped 4 previous similar messages [1702319.260686] Lustre: fir-OST0009: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 99 known clients (2 recovered, 4 in progress, and 0 evicted) already passed deadline 0:05 [1702319.279877] Lustre: Skipped 2 previous similar messages [1702323.887404] Lustre: fir-OST0005: Recovery already passed deadline 0:44, It is most likely due to DNE recovery is failed or stuck, please wait a few more minutes or abort the recovery. [1702323.978338] Lustre: fir-OST0005: Recovery over after 0:45, of 106 clients 106 recovered and 0 were evicted. [1702328.962938] Lustre: fir-OST0001: Recovery over after 0:45, of 107 clients 107 recovered and 0 were evicted. [1702331.602758] Lustre: fir-OST0009: Recovery over after 0:18, of 99 clients 99 recovered and 0 were evicted. [1702343.704175] Lustre: fir-OST0009: Connection restored to fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) [1702343.713573] Lustre: Skipped 365 previous similar messages [1702439.257256] Lustre: fir-OST000b: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [1702439.266650] Lustre: Skipped 8 previous similar messages [1703315.838083] Lustre: fir-OST000b: Connection restored to be50cea3-731d-72c8-23b8-02214076be2e (at 10.8.12.1@o2ib6) [1703315.848547] Lustre: Skipped 18 previous similar messages [1703315.888726] Lustre: fir-OST0005: deleting orphan objects from 0x0:678267 to 0x0:678305 [1703315.888770] Lustre: fir-OST0009: deleting orphan objects from 0x0:678300 to 0x0:678337 [1703315.894095] Lustre: fir-OST0007: deleting orphan objects from 0x0:678301 to 0x0:678337 [1703315.900252] Lustre: fir-OST000b: deleting orphan objects from 0x0:678394 to 0x0:678433 [1703315.903008] Lustre: fir-OST0001: deleting orphan objects from 0x0:678333 to 0x0:678369 [1703315.904004] Lustre: fir-OST0003: deleting orphan objects from 0x0:678192 to 0x0:678241 [1703315.958169] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:477844 to 0xbc0000402:477953 [1703315.958174] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:478001 to 0x540000400:478113 [1703315.958211] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:478007 to 0x800000400:478209 [1703315.967724] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:477875 to 0x380000400:477921 [1703315.972232] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:478326 to 0xb40000402:478433 [1703315.979386] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:477852 to 0xb00000402:477889 [1703317.169774] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000400:147491 to 0xbc0000400:147585 [1703317.169812] Lustre: fir-OST0001: deleting orphan objects from 0xb40000400:147440 to 0xb40000400:147553 [1703317.170675] Lustre: fir-OST0009: deleting orphan objects from 0x540000401:147424 to 0x540000401:147521 [1703317.170695] Lustre: fir-OST0005: deleting orphan objects from 0x800000401:147471 to 0x800000401:147521 [1703317.180947] Lustre: fir-OST000b: deleting orphan objects from 0x380000401:147502 to 0x380000401:147521 [1703317.191630] Lustre: fir-OST0007: deleting orphan objects from 0xb00000400:147421 to 0xb00000400:147457 [1703317.208717] Lustre: fir-OST0007: deleting orphan objects from 0xb00000401:135906 to 0xb00000401:135937 [1703317.210091] Lustre: fir-OST0001: deleting orphan objects from 0xb40000401:135762 to 0xb40000401:135841 [1703317.221409] Lustre: fir-OST0005: deleting orphan objects from 0x800000402:135777 to 0x800000402:135809 [1703317.222424] Lustre: fir-OST000b: deleting orphan objects from 0x380000402:135820 to 0x380000402:135873 [1703317.225238] Lustre: fir-OST0009: deleting orphan objects from 0x540000402:135951 to 0x540000402:135969 [1703317.226169] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000401:135911 to 0xbc0000401:135937 [1703331.929687] Lustre: fir-OST0003: Connection restored to 56a7901d-9f1f-93d3-c75e-92d9b6eaca50 (at 10.9.107.48@o2ib4) [1703331.940316] Lustre: Skipped 2084 previous similar messages [1703365.065934] Lustre: fir-OST0007: Connection restored to d3b31001-f420-4435-d40a-bc3c2a4e6f40 (at 10.8.2.13@o2ib6) [1703365.076408] Lustre: Skipped 308 previous similar messages [1703429.824203] Lustre: fir-OST0003: Connection restored to 01d5ada9-338b-fc1a-1541-7fe86bd87ddc (at 10.9.105.18@o2ib4) [1703429.834812] Lustre: Skipped 118 previous similar messages [1703557.886997] Lustre: fir-OST0005: Connection restored to (at 10.9.103.32@o2ib4) [1703557.894488] Lustre: Skipped 397 previous similar messages [1703682.117265] Lustre: fir-OST0007: haven't heard from client d9d906b5-22c6-448d-c0e5-c1df58bf3870 (at 10.8.31.4@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90aedb3a7c00, cur 1549481398 expire 1549481248 last 1549481171 [1703695.107048] Lustre: fir-OST0009: haven't heard from client d9d906b5-22c6-448d-c0e5-c1df58bf3870 (at 10.8.31.4@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90cb48e41000, cur 1549481411 expire 1549481261 last 1549481184 [1703695.128867] Lustre: Skipped 1 previous similar message [1703815.005849] Lustre: fir-OST0001: Connection restored to bd16151d-af0e-00df-69f0-bc73398a9c87 (at 10.8.4.5@o2ib6) [1703815.016233] Lustre: Skipped 736 previous similar messages [1704327.221013] Lustre: fir-OST0001: Connection restored to db45ffdf-edf3-91b4-c0ad-abe30f1ea215 (at 10.8.27.6@o2ib6) [1704327.231452] Lustre: Skipped 1123 previous similar messages [1704656.129925] Lustre: fir-OST0009: haven't heard from client f6aefb2b-cc93-9031-e9bd-7f3143d0a97d (at 10.8.11.9@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90c6221f2c00, cur 1549482372 expire 1549482222 last 1549482145 [1704656.151717] Lustre: Skipped 2 previous similar messages [1704927.517444] Lustre: fir-OST0009: Connection restored to (at 10.8.8.8@o2ib6) [1704927.524679] Lustre: Skipped 1145 previous similar messages [1705528.043087] Lustre: fir-OST0001: Connection restored to dc527ec4-db62-14c0-459e-fcb26c78037c (at 10.9.107.8@o2ib4) [1705528.053608] Lustre: Skipped 949 previous similar messages [1706128.796900] Lustre: fir-OST000b: Connection restored to eb4e1d20-3a1f-d68d-546e-5c1cf1ecb74b (at 10.9.104.11@o2ib4) [1706128.807505] Lustre: Skipped 1041 previous similar messages [1706206.583857] Lustre: fir-OST0003: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706206.593341] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:479302 to 0xbc0000402:479553 [1706212.022718] Lustre: fir-OST0009: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706224.461216] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:479458 to 0x540000400:479585 [1706224.779315] Lustre: fir-OST0001: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706228.350320] Lustre: fir-OST0001: deleting orphan objects from 0x0:681570 to 0x0:681729 [1706230.625860] Lustre: fir-OST0005: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706235.974089] Lustre: fir-OST0007: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706236.215399] Lustre: fir-OST0005: deleting orphan objects from 0x0:681459 to 0x0:681537 [1706238.578240] Lustre: fir-OST0007: deleting orphan objects from 0x0:681543 to 0x0:681633 [1706254.775717] LNetError: 95014:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 1 seconds [1706254.785895] LNetError: 95014:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (7): c: 0, oc: 1, rc: 8 [1706262.775924] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 7 seconds [1706262.786180] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 8 previous similar messages [1706262.795723] Lustre: 95080:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549483978/real 1549483978] req@ffff90dbee4db600 x1624747513854592/t0(0) o400->fir-MDT0000-lwp-OST0003@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484070 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706262.795743] Lustre: fir-MDT0000-lwp-OST0001: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706262.840894] Lustre: 95080:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1706264.776009] Lustre: 95088:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549483978/real 1549483980] req@ffff90dbee4db000 x1624747513854720/t0(0) o400->fir-MDT0000-lwp-OST0007@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484070 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706264.776020] Lustre: fir-MDT0000-lwp-OST0009: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706264.776022] Lustre: Skipped 5 previous similar messages [1706264.826473] Lustre: 95088:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1706265.775997] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 10 seconds [1706265.786338] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 9 previous similar messages [1706266.776089] Lustre: 95091:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549483978/real 1549483982] req@ffff90dbee4dbc00 x1624747513854384/t0(0) o400->fir-MDT0002-lwp-OST000b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484070 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706266.805015] Lustre: fir-MDT0002-lwp-OST000b: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706266.821176] Lustre: Skipped 3 previous similar messages [1706267.776094] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1706272.776187] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1706272.786449] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [1706318.777357] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 41 seconds [1706318.787706] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1706379.782889] Lustre: fir-OST000b: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706379.783247] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:479458 to 0x540000400:479617 [1706379.783422] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:479552 to 0x800000400:479681 [1706379.783641] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:479555 to 0xbc0000402:479585 [1706379.783817] Lustre: fir-OST000b: deleting orphan objects from 0x0:681652 to 0x0:681729 [1706379.783890] Lustre: fir-OST0009: deleting orphan objects from 0x0:681566 to 0x0:681601 [1706379.783964] Lustre: fir-OST0005: deleting orphan objects from 0x0:681545 to 0x0:681569 [1706379.784060] Lustre: fir-OST0007: deleting orphan objects from 0x0:681543 to 0x0:681633 [1706379.784221] Lustre: fir-OST0001: deleting orphan objects from 0x0:681735 to 0x0:681761 [1706379.784231] Lustre: fir-OST0003: deleting orphan objects from 0x0:681417 to 0x0:681569 [1706379.869162] Lustre: Skipped 11 previous similar messages [1706505.126473] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:479241 to 0xb00000402:479297 [1706505.126502] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:479778 to 0xb40000402:479841 [1706669.128522] Lustre: fir-MDT0002-lwp-OST000b: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706669.144680] Lustre: Skipped 11 previous similar messages [1706669.786359] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 8 seconds [1706669.796614] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [1706669.806218] Lustre: 95080:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549484384/real 1549484385] req@ffff90d45b53dd00 x1624747514490320/t0(0) o400->fir-MDT0001-lwp-OST0003@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484647 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706669.835157] Lustre: 95080:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [1706674.786513] Lustre: 95085:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549484384/real 1549484390] req@ffff90d45b53ad00 x1624747514490208/t0(0) o400->fir-MDT0003-lwp-OST0005@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484647 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706674.786528] Lustre: fir-MDT0003-lwp-OST000b: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706674.786530] Lustre: Skipped 11 previous similar messages [1706674.837052] Lustre: 95085:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [1706702.787190] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1706702.797447] LNet: 95014:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 28 previous similar messages [1706713.335787] Lustre: Failing over fir-OST000b [1706713.340241] Lustre: Skipped 5 previous similar messages [1706713.358196] LustreError: 20305:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST0009_UUID: namespace resource [0x540000400:0x74d2b:0x0].0x0 (ffff90a08cd88780) refcount nonzero (2) after lock cleanup; forcing cleanup. [1706713.391882] Lustre: fir-OST0003: Not available for connect from 10.9.102.52@o2ib4 (stopping) [1706713.400504] Lustre: Skipped 4 previous similar messages [1706713.907504] Lustre: fir-OST0001: Not available for connect from 10.9.101.22@o2ib4 (stopping) [1706713.916124] Lustre: Skipped 62 previous similar messages [1706714.915675] Lustre: fir-OST000b: Not available for connect from 10.9.101.24@o2ib4 (stopping) [1706714.924295] Lustre: Skipped 84 previous similar messages [1706715.652256] LustreError: 137-5: fir-OST0009_UUID: not available for connect from 10.8.7.30@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706715.669647] LustreError: Skipped 351 previous similar messages [1706716.199026] Lustre: server umount fir-OST0009 complete [1706716.204352] Lustre: Skipped 2 previous similar messages [1706721.156836] md7: detected capacity change from 64011422924800 to 0 [1706721.163217] md: md7 stopped. [1706721.194206] md5: detected capacity change from 64011422924800 to 0 [1706721.200571] md: md5 stopped. [1706721.267194] md9: detected capacity change from 64011422924800 to 0 [1706721.273558] md: md9 stopped. [1706721.293777] md1: detected capacity change from 64011422924800 to 0 [1706721.300149] md: md1 stopped. [1706723.688778] md3: detected capacity change from 64011422924800 to 0 [1706723.695146] md: md3 stopped. [1706724.762468] md: md3 stopped. [1706725.463201] md11: detected capacity change from 64011422924800 to 0 [1706725.469658] md: md11 stopped. [1706726.997889] LNetError: 15588:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.201@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1706727.725258] LNetError: 15588:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.210@o2ib7 on NA (ib0:1:10.0.10.102): bad dst nid 10.0.10.102@o2ib7 [1706727.740908] LNetError: 15588:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Skipped 15 previous similar messages [1706728.998848] LNet: Removed LNI 10.0.10.102@o2ib7 [1706762.469051] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1706762.476952] alg: No test for adler32 (adler32-zlib) [1706763.280733] Lustre: Lustre: Build Version: 2.12.0 [1706763.406682] LNet: Using FastReg for registration [1706763.424251] LNet: Added LNI 10.0.10.102@o2ib7 [8/256/0/180] [1706766.302279] md: md1 stopped. [1706766.357190] md/raid:md1: device dm-26 operational as raid disk 0 [1706766.363391] md/raid:md1: device dm-9 operational as raid disk 9 [1706766.369492] md/raid:md1: device dm-16 operational as raid disk 8 [1706766.375682] md/raid:md1: device dm-59 operational as raid disk 7 [1706766.381882] md/raid:md1: device dm-23 operational as raid disk 6 [1706766.388069] md/raid:md1: device dm-60 operational as raid disk 5 [1706766.394257] md/raid:md1: device dm-34 operational as raid disk 4 [1706766.400463] md/raid:md1: device dm-68 operational as raid disk 3 [1706766.406648] md/raid:md1: device dm-43 operational as raid disk 2 [1706766.412845] md/raid:md1: device dm-57 operational as raid disk 1 [1706766.419851] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706766.444987] md1: detected capacity change from 0 to 64011422924800 [1706766.464151] md: md5 stopped. [1706766.527576] md/raid:md5: device dm-84 operational as raid disk 0 [1706766.533765] md/raid:md5: device dm-8 operational as raid disk 9 [1706766.539865] md/raid:md5: device dm-3 operational as raid disk 8 [1706766.545969] md/raid:md5: device dm-22 operational as raid disk 7 [1706766.552155] md/raid:md5: device dm-21 operational as raid disk 6 [1706766.558352] md/raid:md5: device dm-67 operational as raid disk 5 [1706766.564543] md/raid:md5: device dm-36 operational as raid disk 4 [1706766.570738] md/raid:md5: device dm-10 operational as raid disk 3 [1706766.576925] md/raid:md5: device dm-76 operational as raid disk 2 [1706766.583113] md/raid:md5: device dm-56 operational as raid disk 1 [1706766.590144] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706766.621258] md5: detected capacity change from 0 to 64011422924800 [1706766.630279] md: md7 stopped. [1706766.659477] md/raid:md7: device dm-92 operational as raid disk 0 [1706766.665672] md/raid:md7: device dm-38 operational as raid disk 9 [1706766.671864] md/raid:md7: device dm-49 operational as raid disk 8 [1706766.678051] md/raid:md7: device dm-97 operational as raid disk 7 [1706766.684241] md/raid:md7: device dm-111 operational as raid disk 6 [1706766.690515] md/raid:md7: device dm-96 operational as raid disk 5 [1706766.696707] md/raid:md7: device dm-101 operational as raid disk 4 [1706766.702983] md/raid:md7: device dm-91 operational as raid disk 3 [1706766.709172] md/raid:md7: device dm-86 operational as raid disk 2 [1706766.715357] md/raid:md7: device dm-109 operational as raid disk 1 [1706766.722650] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706766.740982] md7: detected capacity change from 0 to 64011422924800 [1706766.750952] md: md9 stopped. [1706766.772534] md/raid:md9: device dm-47 operational as raid disk 0 [1706766.778727] md/raid:md9: device dm-42 operational as raid disk 9 [1706766.784915] md/raid:md9: device dm-39 operational as raid disk 8 [1706766.791100] md/raid:md9: device dm-28 operational as raid disk 7 [1706766.797289] md/raid:md9: device dm-116 operational as raid disk 6 [1706766.803564] md/raid:md9: device dm-71 operational as raid disk 5 [1706766.809755] md/raid:md9: device dm-110 operational as raid disk 4 [1706766.816031] md/raid:md9: device dm-94 operational as raid disk 3 [1706766.822229] md/raid:md9: device dm-33 operational as raid disk 2 [1706766.828432] md/raid:md9: device dm-53 operational as raid disk 1 [1706766.835669] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706766.858335] md9: detected capacity change from 0 to 64011422924800 [1706766.870454] md: md11 stopped. [1706766.956258] md/raid:md11: device dm-50 operational as raid disk 0 [1706766.962563] md/raid:md11: device dm-46 operational as raid disk 9 [1706766.968906] md/raid:md11: device dm-30 operational as raid disk 8 [1706766.975253] md/raid:md11: device dm-112 operational as raid disk 7 [1706766.981622] md/raid:md11: device dm-114 operational as raid disk 6 [1706766.987989] md/raid:md11: device dm-80 operational as raid disk 5 [1706766.994261] md/raid:md11: device dm-48 operational as raid disk 4 [1706767.000535] md/raid:md11: device dm-40 operational as raid disk 3 [1706767.006817] md/raid:md11: device dm-104 operational as raid disk 2 [1706767.013186] md/raid:md11: device dm-95 operational as raid disk 1 [1706767.020413] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706767.069610] md11: detected capacity change from 0 to 64011422924800 [1706767.097653] md: md3 stopped. [1706767.145470] md/raid:md3: device dm-2 operational as raid disk 0 [1706767.151593] md/raid:md3: device dm-0 operational as raid disk 9 [1706767.157702] md/raid:md3: device dm-65 operational as raid disk 8 [1706767.163898] md/raid:md3: device dm-24 operational as raid disk 7 [1706767.170095] md/raid:md3: device dm-14 operational as raid disk 6 [1706767.176290] md/raid:md3: device dm-118 operational as raid disk 5 [1706767.182577] md/raid:md3: device dm-64 operational as raid disk 4 [1706767.188772] md/raid:md3: device dm-75 operational as raid disk 3 [1706767.194968] md/raid:md3: device dm-31 operational as raid disk 2 [1706767.201167] md/raid:md3: device dm-15 operational as raid disk 1 [1706767.208437] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706767.228889] md3: detected capacity change from 0 to 64011422924800 [1706767.314121] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1706767.654088] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706767.654042] LDISKFS-fs (md5): file extents enabled [1706767.657036] LDISKFS-fs (md7): file extents enabled [1706767.657037] , maximum tree depth=5 [1706767.676620] , maximum tree depth=5 [1706767.977615] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706768.031048] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1706768.178804] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706768.397622] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706768.583857] LDISKFS-fs (md3): file extents enabled [1706768.585040] LDISKFS-fs (md11): file extents enabled [1706768.585041] , maximum tree depth=5 [1706768.595637] , maximum tree depth=5 [1706768.921688] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706768.939730] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706773.406001] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1706773.441543] LustreError: 137-5: fir-OST0007_UUID: not available for connect from 10.8.21.8@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706773.960582] Lustre: fir-OST0001: Not available for connect from 10.8.6.5@o2ib6 (not set up) [1706773.960587] LustreError: 137-5: fir-OST0009_UUID: not available for connect from 10.8.6.5@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706773.960589] LustreError: Skipped 39 previous similar messages [1706774.470898] Lustre: fir-OST0001: Not available for connect from 10.8.26.6@o2ib6 (not set up) [1706774.479518] Lustre: Skipped 6 previous similar messages [1706774.962260] LustreError: 137-5: fir-OST0000_UUID: not available for connect from 10.8.7.26@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706774.979669] LustreError: Skipped 109 previous similar messages [1706776.977258] LustreError: 137-5: fir-OST0003_UUID: not available for connect from 10.9.102.37@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706776.994800] LustreError: Skipped 233 previous similar messages [1706781.003669] LustreError: 137-5: fir-OST0005_UUID: not available for connect from 10.8.8.23@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706781.021052] LustreError: Skipped 231 previous similar messages [1706781.406199] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1706782.406227] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1706784.406276] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 4 seconds [1706784.416530] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1706784.428044] Lustre: fir-OST0001: Imperative Recovery not enabled, recovery window 300-900 [1706784.437024] Lustre: fir-OST0001: in recovery but waiting for the first client to connect [1706784.496737] Lustre: fir-OST0001: Will be in recovery for at least 5:00, or until 503 clients reconnect [1706784.503825] Lustre: fir-OST0001: Connection restored to (at 10.8.2.3@o2ib6) [1706784.727243] Lustre: fir-OST0007: Not available for connect from 10.8.8.25@o2ib6 (not set up) [1706784.735868] Lustre: Skipped 7 previous similar messages [1706785.013324] Lustre: fir-OST0001: Connection restored to (at 10.9.101.41@o2ib4) [1706785.020811] Lustre: Skipped 14 previous similar messages [1706786.108089] Lustre: fir-OST0001: Connection restored to (at 10.9.101.56@o2ib4) [1706786.115580] Lustre: Skipped 27 previous similar messages [1706788.149537] Lustre: fir-OST0001: Connection restored to (at 10.8.26.18@o2ib6) [1706788.156937] Lustre: Skipped 42 previous similar messages [1706788.406381] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 8 seconds [1706788.416634] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1706789.029980] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.8.25.14@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706789.047458] LustreError: Skipped 413 previous similar messages [1706792.170706] Lustre: fir-OST0001: Connection restored to (at 10.8.20.28@o2ib6) [1706792.178111] Lustre: Skipped 173 previous similar messages [1706795.409038] Lustre: fir-OST0007: Imperative Recovery not enabled, recovery window 300-900 [1706795.418108] Lustre: fir-OST0007: in recovery but waiting for the first client to connect [1706795.419247] Lustre: fir-OST0007: Will be in recovery for at least 5:00, or until 508 clients reconnect [1706795.745825] Lustre: fir-OST0005: Not available for connect from 10.9.107.37@o2ib4 (not set up) [1706796.406601] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1706796.416862] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1706800.218515] Lustre: fir-OST0007: Connection restored to (at 10.8.11.13@o2ib6) [1706800.225918] Lustre: Skipped 477 previous similar messages [1706800.409558] Lustre: fir-OST0005: Imperative Recovery not enabled, recovery window 300-900 [1706800.418565] Lustre: fir-OST0005: in recovery but waiting for the first client to connect [1706800.427276] Lustre: fir-OST0005: Will be in recovery for at least 5:00, or until 509 clients reconnect [1706805.111281] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.8.25.10@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706805.128758] LustreError: Skipped 1417 previous similar messages [1706805.409805] Lustre: fir-OST0009: Imperative Recovery not enabled, recovery window 300-900 [1706805.418782] Lustre: fir-OST0009: in recovery but waiting for the first client to connect [1706805.419674] Lustre: fir-OST0009: Will be in recovery for at least 5:00, or until 506 clients reconnect [1706808.832113] Lustre: fir-OST0001: Recovery over after 0:24, of 503 clients 503 recovered and 0 were evicted. [1706809.359204] Lustre: fir-OST0007: Recovery over after 0:14, of 508 clients 508 recovered and 0 were evicted. [1706810.409321] Lustre: fir-OST0003: Imperative Recovery not enabled, recovery window 300-900 [1706810.418776] Lustre: fir-OST0003: in recovery but waiting for the first client to connect [1706810.420578] Lustre: fir-OST0003: Will be in recovery for at least 5:00, or until 479 clients reconnect [1706812.057711] Lustre: fir-OST0009: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 506 known clients (310 recovered, 106 in progress, and 0 evicted) already passed deadline 0:06 [1706812.407005] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 18 seconds [1706812.417350] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [1706813.117575] Lustre: fir-OST0005: Recovery over after 0:12, of 509 clients 509 recovered and 0 were evicted. [1706814.912820] Lustre: fir-OST0003: Denying connection for new client afd89a08-6b67-04f7-2303-d504e382d3cd(at 10.9.105.61@o2ib4), waiting for 479 known clients (195 recovered, 78 in progress, and 0 evicted) already passed deadline 0:04 [1706814.933591] Lustre: Skipped 3 previous similar messages [1706816.253661] Lustre: fir-OST0003: Connection restored to e9fb76c4-12e4-0a58-01e7-cbc64e55ba4c (at 10.8.23.36@o2ib6) [1706816.264198] Lustre: Skipped 1636 previous similar messages [1706818.478064] Lustre: fir-OST0003: Denying connection for new client afd89a08-6b67-04f7-2303-d504e382d3cd(at 10.9.105.61@o2ib4), waiting for 479 known clients (316 recovered, 109 in progress, and 0 evicted) already passed deadline 0:07 [1706820.410078] Lustre: fir-OST000b: Imperative Recovery not enabled, recovery window 300-900 [1706820.419064] Lustre: fir-OST000b: in recovery but waiting for the first client to connect [1706820.444618] Lustre: fir-OST000b: Will be in recovery for at least 5:00, or until 506 clients reconnect [1706823.240231] Lustre: fir-OST000b: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 506 known clients (141 recovered, 61 in progress, and 0 evicted) already passed deadline 0:02 [1706824.370139] Lustre: fir-OST0009: Recovery over after 0:19, of 506 clients 506 recovered and 0 were evicted. [1706828.428464] Lustre: fir-OST000b: Denying connection for new client afd89a08-6b67-04f7-2303-d504e382d3cd(at 10.9.105.61@o2ib4), waiting for 506 known clients (342 recovered, 127 in progress, and 0 evicted) already passed deadline 0:07 [1706833.489232] Lustre: fir-OST000b: Recovery over after 0:13, of 506 clients 506 recovered and 0 were evicted. [1706839.976221] Lustre: fir-OST0003: Denying connection for new client afd89a08-6b67-04f7-2303-d504e382d3cd(at 10.9.105.61@o2ib4), waiting for 479 known clients (365 recovered, 113 in progress, and 0 evicted) already passed deadline 0:29 [1706839.997086] Lustre: Skipped 16 previous similar messages [1706849.541613] Lustre: fir-OST000b: Connection restored to (at 10.8.25.27@o2ib6) [1706849.549053] Lustre: Skipped 672 previous similar messages [1706854.486876] Lustre: fir-OST0003: Recovery over after 0:44, of 479 clients 479 recovered and 0 were evicted. [1706882.231331] Lustre: fir-OST0001: deleting orphan objects from 0xb40000400:148591 to 0xb40000400:148609 [1706882.231397] Lustre: fir-OST0009: deleting orphan objects from 0x540000401:148563 to 0x540000401:148609 [1706882.233582] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000400:148622 to 0xbc0000400:148641 [1706882.253093] Lustre: fir-OST000b: deleting orphan objects from 0x380000401:148567 to 0x380000401:148609 [1706882.258077] Lustre: fir-OST0005: deleting orphan objects from 0x800000401:148564 to 0x800000401:148609 [1706882.290635] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000401:137867 to 0xbc0000401:137889 [1706882.296321] Lustre: fir-OST000b: deleting orphan objects from 0x380000402:137806 to 0x380000402:137825 [1706882.306950] Lustre: fir-OST0001: deleting orphan objects from 0xb40000401:137771 to 0xb40000401:137793 [1706882.317247] Lustre: fir-OST0009: deleting orphan objects from 0x540000402:137898 to 0x540000402:137921 [1706882.340729] Lustre: fir-OST0007: deleting orphan objects from 0xb00000400:148500 to 0xb00000400:148545 [1706882.342885] Lustre: fir-OST0007: deleting orphan objects from 0xb00000401:137855 to 0xb00000401:137889 [1706882.465161] Lustre: fir-OST0005: deleting orphan objects from 0x800000402:137739 to 0x800000402:137761 [1706951.603309] Lustre: fir-OST0001: Connection restored to (at 10.8.1.30@o2ib6) [1706951.610635] Lustre: Skipped 64 previous similar messages [1706951.666372] Lustre: fir-OST0001: deleting orphan objects from 0x0:681876 to 0x0:681921 [1706951.666876] Lustre: fir-OST0009: deleting orphan objects from 0x0:681716 to 0x0:681761 [1706951.679714] Lustre: fir-OST0005: deleting orphan objects from 0x0:681683 to 0x0:681729 [1706951.681572] Lustre: fir-OST0003: deleting orphan objects from 0x0:681681 to 0x0:681697 [1706951.681911] Lustre: fir-OST0007: deleting orphan objects from 0x0:681747 to 0x0:681793 [1706951.682970] Lustre: fir-OST000b: deleting orphan objects from 0x0:681845 to 0x0:681889 [1706972.312436] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:479591 to 0xbc0000402:479617 [1706972.312470] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:479690 to 0x800000400:479713 [1706972.312494] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:479625 to 0x540000400:479649 [1706972.320992] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:479304 to 0xb00000402:479329 [1706972.338300] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:479848 to 0xb40000402:479873 [1706972.343001] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:479269 to 0x380000400:479329 [1707079.714167] Lustre: fir-OST0003: Connection restored to (at 10.8.25.31@o2ib6) [1707079.721573] Lustre: Skipped 1359 previous similar messages [1707336.310793] Lustre: fir-OST000b: Connection restored to 3ba4f6ab-da5b-8d5b-7e10-cb73b415cd02 (at 10.8.3.11@o2ib6) [1707336.321236] Lustre: Skipped 486 previous similar messages [1707850.900838] Lustre: fir-OST0001: Connection restored to (at 10.8.6.4@o2ib6) [1707850.908066] Lustre: Skipped 976 previous similar messages [1708451.241735] Lustre: fir-OST0003: Connection restored to (at 10.9.103.27@o2ib4) [1708451.249226] Lustre: Skipped 1019 previous similar messages [1709107.017768] Lustre: fir-OST0005: Connection restored to (at 10.8.1.24@o2ib6) [1709107.025082] Lustre: Skipped 171 previous similar messages [1709727.208520] Lustre: fir-OST0003: Connection restored to f3bddf34-379e-90f9-b8a3-37dd2323157d (at 10.8.3.18@o2ib6) [1709727.218963] Lustre: Skipped 42 previous similar messages [1710341.606698] Lustre: fir-OST0001: Connection restored to b5d0fd65-06a7-fd14-4925-ef95dfe63868 (at 10.9.107.28@o2ib4) [1710341.617307] Lustre: Skipped 41 previous similar messages [1710770.574303] Lustre: 25044:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488479/real 1549488479] req@ffff90a670f53f00 x1624752238941376/t0(0) o106->fir-OST0005@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488486 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1710770.574305] Lustre: 26508:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488479/real 1549488479] req@ffff90a670f55100 x1624752238941424/t0(0) o106->fir-OST0007@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488486 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1710770.629223] Lustre: 25044:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1710777.574483] Lustre: 26508:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488486/real 1549488486] req@ffff90a670f55100 x1624752238941424/t0(0) o106->fir-OST0007@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488493 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710777.602065] Lustre: 26508:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1710784.575667] Lustre: 26480:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488493/real 1549488493] req@ffff90a152d61500 x1624752238941456/t0(0) o106->fir-OST0001@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488500 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710784.575670] Lustre: 26505:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488493/real 1549488493] req@ffff90a670f55d00 x1624752238941504/t0(0) o106->fir-OST0003@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488500 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710784.575674] Lustre: 26505:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1710784.640422] Lustre: 26480:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [1710791.575840] Lustre: 26505:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488500/real 1549488500] req@ffff90a670f55d00 x1624752238941504/t0(0) o106->fir-OST0003@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488507 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710798.604020] Lustre: 26505:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488507/real 1549488507] req@ffff90a670f55d00 x1624752238941504/t0(0) o106->fir-OST0003@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488514 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710798.631475] Lustre: 26505:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1710812.612379] Lustre: 26508:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488521/real 1549488521] req@ffff90a670f55100 x1624752238941424/t0(0) o106->fir-OST0007@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488528 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710812.639985] Lustre: 26508:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [1710833.639917] Lustre: 25044:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488542/real 1549488542] req@ffff90a670f53f00 x1624752238941376/t0(0) o106->fir-OST0005@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488549 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710833.667833] Lustre: 25044:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 12 previous similar messages [1710868.650812] Lustre: 26508:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549488577/real 1549488577] req@ffff90a670f55100 x1624752238941424/t0(0) o106->fir-OST0007@10.8.27.23@o2ib6:15/16 lens 296/280 e 0 to 1 dl 1549488584 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1710868.678256] Lustre: 26508:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 18 previous similar messages [1710882.302219] Lustre: fir-OST0001: haven't heard from client d2739644-8a91-79ff-73b1-4208fca7fb94 (at 10.8.27.23@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90aea20b9800, cur 1549488598 expire 1549488448 last 1549488371 [1710892.287035] Lustre: fir-OST0005: haven't heard from client d2739644-8a91-79ff-73b1-4208fca7fb94 (at 10.8.27.23@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff909bcafb5c00, cur 1549488608 expire 1549488458 last 1549488381 [1710892.308914] Lustre: Skipped 1 previous similar message [1710942.050272] Lustre: fir-OST0007: Connection restored to (at 10.8.17.1@o2ib6) [1710942.057710] Lustre: Skipped 923 previous similar messages [1711175.359099] Lustre: fir-OST0005: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1711176.540953] Lustre: fir-OST0005: deleting orphan objects from 0x0:682702 to 0x0:682721 [1711189.953313] Lustre: fir-OST0009: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1711190.655494] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:480610 to 0x540000400:480673 [1711191.367371] Lustre: fir-OST0007: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1711198.273095] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:480283 to 0xb00000402:480449 [1711207.315874] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1711207.333341] LustreError: Skipped 535 previous similar messages [1711299.306635] Lustre: fir-OST0005: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1711304.115916] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:480667 to 0x800000400:480737 [1711547.240430] Lustre: fir-OST0003: Connection restored to (at 10.9.105.21@o2ib4) [1711547.247921] Lustre: Skipped 820 previous similar messages [1712147.502391] Lustre: fir-OST0005: Connection restored to 05d8ca20-c344-bd96-2627-67efcff39463 (at 10.9.101.7@o2ib4) [1712147.512920] Lustre: Skipped 770 previous similar messages [1712288.323119] Lustre: fir-OST0003: haven't heard from client d30b0f27-8a20-f73b-bae5-225c0f2eca1e (at 10.8.2.19@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90a2d3a1c400, cur 1549490004 expire 1549489854 last 1549489777 [1712288.344918] Lustre: Skipped 3 previous similar messages [1712750.114164] Lustre: fir-OST0005: Connection restored to (at 10.8.18.23@o2ib6) [1712750.121567] Lustre: Skipped 905 previous similar messages [1712767.348088] Lustre: fir-OST0009: haven't heard from client 448ba54c-6ace-4879-39fb-12a7e98699b7 (at 10.8.11.9@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90c68442e400, cur 1549490483 expire 1549490333 last 1549490256 [1712767.369909] Lustre: Skipped 4 previous similar messages [1713351.123013] Lustre: fir-OST0001: Connection restored to (at 10.8.13.28@o2ib6) [1713351.130424] Lustre: Skipped 1126 previous similar messages [1713392.349490] Lustre: fir-OST0005: haven't heard from client c67a7bb7-66c6-d9cf-bfce-24d6fb1c7987 (at 10.8.11.9@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90ae3337e400, cur 1549491108 expire 1549490958 last 1549490881 [1713392.371275] Lustre: Skipped 5 previous similar messages [1713951.452230] Lustre: fir-OST0007: Connection restored to (at 10.9.106.70@o2ib4) [1713951.459715] Lustre: Skipped 1024 previous similar messages [1714304.372160] Lustre: fir-OST0007: haven't heard from client 09620a96-bb25-f006-9497-13a0261dca16 (at 10.8.27.23@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90b0ae2bc400, cur 1549492020 expire 1549491870 last 1549491793 [1714304.394049] Lustre: Skipped 5 previous similar messages [1714552.177439] Lustre: fir-OST0001: Connection restored to 7e4f58d8-6e8c-32f9-59fb-7de9af0df4b4 (at 10.9.106.12@o2ib4) [1714552.188051] Lustre: Skipped 837 previous similar messages [1715153.244574] Lustre: fir-OST0007: Connection restored to (at 10.8.18.22@o2ib6) [1715153.251978] Lustre: Skipped 671 previous similar messages [1715560.403980] Lustre: fir-OST0007: haven't heard from client 21210b5c-1bf9-2672-2025-9c6097d35430 (at 10.9.103.25@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90abdf826000, cur 1549493276 expire 1549493126 last 1549493049 [1715560.425941] Lustre: Skipped 3 previous similar messages [1715753.528833] Lustre: fir-OST0001: Connection restored to 77244e83-b250-397d-f179-1d8851db1705 (at 10.8.12.2@o2ib6) [1715753.539267] Lustre: Skipped 892 previous similar messages [1716356.708337] Lustre: fir-OST000b: Connection restored to (at 10.9.105.69@o2ib4) [1716356.715831] Lustre: Skipped 950 previous similar messages [1716957.680255] Lustre: fir-OST0005: Connection restored to (at 10.8.27.4@o2ib6) [1716957.687568] Lustre: Skipped 872 previous similar messages [1717128.236461] Lustre: fir-OST0001: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1717131.424390] Lustre: fir-OST0001: deleting orphan objects from 0x0:683470 to 0x0:683553 [1717144.526669] Lustre: fir-OST0007: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1717175.910215] Lustre: fir-OST0007: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1717181.545705] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1717181.563164] LustreError: Skipped 1 previous similar message [1717183.446102] LustreError: 137-5: fir-OST000a_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1717183.463561] LustreError: Skipped 1 previous similar message [1717183.473030] Lustre: fir-OST000b: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1717183.665516] Lustre: fir-OST0001: deleting orphan objects from 0x0:683470 to 0x0:683585 [1717183.812702] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:483426 to 0x380000400:483553 [1717183.812810] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:483570 to 0xb00000402:483649 [1717184.541999] LustreError: 137-5: fir-OST0008_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1717184.559477] LustreError: Skipped 1 previous similar message [1717184.659063] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:483864 to 0x800000400:483905 [1717185.159776] Lustre: fir-OST0009: deleting orphan objects from 0x0:683306 to 0x0:683329 [1717211.187559] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1717211.188062] Lustre: fir-OST0007: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1717211.188064] Lustre: Skipped 3 previous similar messages [1717211.188350] Lustre: fir-OST0007: deleting orphan objects from 0x0:683344 to 0x0:683361 [1717211.227660] LustreError: Skipped 1 previous similar message [1717562.125692] Lustre: fir-OST0001: Connection restored to (at 10.8.18.24@o2ib6) [1717562.133098] Lustre: Skipped 808 previous similar messages [1718162.701190] Lustre: fir-OST0007: Connection restored to (at 10.8.24.7@o2ib6) [1718162.708507] Lustre: Skipped 734 previous similar messages [1718766.772520] Lustre: fir-OST000b: Connection restored to 2196625d-9992-1b8a-5a12-40751a9cdd4e (at 10.9.107.2@o2ib4) [1718766.783043] Lustre: Skipped 732 previous similar messages [1719367.697770] Lustre: fir-OST0001: Connection restored to (at 10.9.106.72@o2ib4) [1719367.705261] Lustre: Skipped 783 previous similar messages [1719571.512056] Lustre: fir-OST0001: haven't heard from client 15c5ad4d-3c8c-9727-45c4-08248f6d4b6e (at 10.8.11.9@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90cbfbc5a400, cur 1549497287 expire 1549497137 last 1549497060 [1719571.533860] Lustre: Skipped 3 previous similar messages [1719967.766714] Lustre: fir-OST0009: Connection restored to (at 10.9.102.4@o2ib4) [1719967.774113] Lustre: Skipped 820 previous similar messages [1720568.776908] Lustre: fir-OST0001: Connection restored to (at 10.8.10.5@o2ib6) [1720568.784227] Lustre: Skipped 761 previous similar messages [1721171.197775] Lustre: fir-OST0005: Connection restored to (at 10.8.27.4@o2ib6) [1721171.205092] Lustre: Skipped 642 previous similar messages [1721773.809150] Lustre: fir-OST000b: Connection restored to (at 10.8.1.2@o2ib6) [1721773.816377] Lustre: Skipped 641 previous similar messages [1722374.400404] Lustre: fir-OST0003: Connection restored to (at 10.8.4.29@o2ib6) [1722374.407718] Lustre: Skipped 600 previous similar messages [1722983.375917] Lustre: fir-OST0003: Connection restored to 3ba4f6ab-da5b-8d5b-7e10-cb73b415cd02 (at 10.8.3.11@o2ib6) [1722983.386352] Lustre: Skipped 635 previous similar messages [1723586.558191] Lustre: fir-OST0001: Connection restored to (at 10.9.107.9@o2ib4) [1723586.565595] Lustre: Skipped 666 previous similar messages [1724080.741516] Lustre: fir-OST0007: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1724085.991850] Lustre: fir-OST0001: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1724085.992347] Lustre: fir-OST0007: deleting orphan objects from 0x0:684277 to 0x0:684321 [1724086.825814] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:485527 to 0xb40000402:485761 [1724094.227848] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:485382 to 0x540000400:485473 [1724100.896694] Lustre: fir-OST0001: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1724100.905831] Lustre: Skipped 2 previous similar messages [1724109.953831] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:485527 to 0xb40000402:485793 [1724138.356572] LustreError: 137-5: fir-OST0004_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1724163.446039] Lustre: fir-OST0001: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1724163.479719] Lustre: fir-OST0001: deleting orphan objects from 0x0:684513 to 0x0:684545 [1724186.671719] Lustre: fir-OST0007: Connection restored to 24cdf384-7a03-f547-7fc8-3ee3b567a27a (at 10.8.11.35@o2ib6) [1724186.682243] Lustre: Skipped 1176 previous similar messages [1724594.652223] Lustre: fir-OST0009: haven't heard from client e3ecb760-8d4d-9ac6-163a-0830c5c51bce (at 10.8.11.22@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90c43ffe2400, cur 1549502310 expire 1549502160 last 1549502083 [1724594.674133] Lustre: Skipped 4 previous similar messages [1724694.762327] Lustre: fir-OST0005: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1724695.820564] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:485871 to 0x800000400:485889 [1724704.189703] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:485871 to 0x800000400:485889 [1724787.004705] Lustre: fir-OST0009: Connection restored to aaf0a12e-0503-9b8d-5b98-b850f09a0ee4 (at 10.9.103.28@o2ib4) [1724787.015311] Lustre: Skipped 716 previous similar messages [1725392.900752] Lustre: fir-OST0001: Connection restored to b9de1fd1-ccbe-721f-e4ab-c6e06447a81c (at 10.8.15.4@o2ib6) [1725392.911190] Lustre: Skipped 442 previous similar messages [1725995.237176] Lustre: fir-OST0005: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1725995.247697] Lustre: Skipped 758 previous similar messages [1726149.677577] Lustre: fir-OST000b: haven't heard from client 1a88bb1c-f105-e166-5f8b-cd9f0187ea3f (at 10.8.11.22@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90a9d36bc400, cur 1549503865 expire 1549503715 last 1549503638 [1726149.699474] Lustre: Skipped 5 previous similar messages [1726597.525854] Lustre: fir-OST0005: Connection restored to (at 10.9.106.6@o2ib4) [1726597.533261] Lustre: Skipped 492 previous similar messages [1727202.509480] Lustre: fir-OST0003: Connection restored to 926fa24d-f3ab-7ad6-dbc7-f8a15bdf8c5a (at 10.8.19.8@o2ib6) [1727202.519920] Lustre: Skipped 429 previous similar messages [1727802.693308] Lustre: fir-OST000b: Connection restored to 8427e8d1-34c3-886c-dab9-5ccc235f41c4 (at 10.9.107.5@o2ib4) [1727802.703829] Lustre: Skipped 275 previous similar messages [1727826.530316] Lustre: fir-OST0003: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1727826.539458] Lustre: Skipped 1 previous similar message [1727828.834521] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:486646 to 0xbc0000402:486785 [1727834.619554] Lustre: fir-OST0005: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1727835.410141] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:486878 to 0x800000400:486913 [1728282.579914] Lustre: fir-OST0003: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1728301.957019] LNetError: 21353:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1728301.967194] LNetError: 21353:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (1): c: 0, oc: 2, rc: 8 [1728301.979905] Lustre: 21380:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549506011/real 1549506017] req@ffff909fd17a9500 x1624752261930688/t0(0) o400->fir-MDT0000-lwp-OST000b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549506089 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1728301.979922] Lustre: fir-MDT0000-lwp-OST0003: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1728302.025043] Lustre: 21380:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 13 previous similar messages [1728302.957054] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [1728302.967308] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1728308.957198] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1728308.967458] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 8 previous similar messages [1728316.102334] Lustre: fir-OST000b: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1728316.102336] Lustre: fir-OST0001: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1728316.102337] Lustre: fir-OST0009: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1728316.102338] Lustre: fir-OST0009: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1728316.127761] Lustre: fir-OST0001: deleting orphan objects from 0x0:685629 to 0x0:685665 [1728316.127813] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:486923 to 0x540000400:486977 [1728316.127898] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:486996 to 0x800000400:487041 [1728316.128046] Lustre: fir-OST0007: deleting orphan objects from 0x0:685403 to 0x0:685441 [1728316.128133] Lustre: fir-OST0005: deleting orphan objects from 0x0:685298 to 0x0:685313 [1728316.128134] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:486687 to 0xb00000402:486721 [1728316.128357] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:487241 to 0xb40000402:487329 [1728316.128359] Lustre: fir-OST0009: deleting orphan objects from 0x0:685327 to 0x0:685345 [1728316.209185] Lustre: Skipped 5 previous similar messages [1728317.125096] Lustre: fir-OST0003: deleting orphan objects from 0x0:685263 to 0x0:685281 [1728317.928282] Lustre: fir-OST000b: deleting orphan objects from 0x0:685441 to 0x0:685473 [1728319.304326] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:486587 to 0x380000400:486657 [1728340.091001] Lustre: 21382:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549506011/real 1549506011] req@ffff909fd17a8900 x1624752261930336/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1549506055 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1728340.119293] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1728373.226779] Lustre: fir-OST0003: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1728373.235958] Lustre: Skipped 1 previous similar message [1728373.241544] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:486870 to 0xbc0000402:486913 [1728374.171869] Lustre: 21388:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549506011/real 1549506011] req@ffff909fd17a9200 x1624752261930432/t0(0) o400->fir-MDT0000-lwp-OST0007@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549506089 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1728374.171876] Lustre: fir-MDT0002-lwp-OST0007: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1728374.171877] Lustre: fir-MDT0000-lwp-OST0001: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1728374.171878] Lustre: Skipped 5 previous similar messages [1728374.171879] Lustre: Skipped 5 previous similar messages [1728374.243620] Lustre: 21388:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1728408.737770] Lustre: fir-OST0001: Connection restored to (at 10.0.10.3@o2ib7) [1728408.745082] Lustre: Skipped 331 previous similar messages [1729008.753400] Lustre: fir-OST0005: Connection restored to (at 10.0.10.3@o2ib7) [1729008.760746] Lustre: Skipped 301 previous similar messages [1729608.769680] Lustre: fir-OST0005: Connection restored to (at 10.0.10.3@o2ib7) [1729608.776998] Lustre: Skipped 285 previous similar messages [1730208.786062] Lustre: fir-OST0007: Connection restored to (at 10.0.10.3@o2ib7) [1730208.793377] Lustre: Skipped 369 previous similar messages [1730809.798930] Lustre: fir-OST0001: Connection restored to (at 10.0.10.3@o2ib7) [1730809.806245] Lustre: Skipped 327 previous similar messages [1731412.722383] Lustre: fir-OST0009: Connection restored to (at 10.9.103.4@o2ib4) [1731412.729787] Lustre: Skipped 351 previous similar messages [1732019.586106] Lustre: fir-OST0001: Connection restored to (at 10.9.107.13@o2ib4) [1732019.593603] Lustre: Skipped 333 previous similar messages [1732129.486287] Lustre: fir-OST0005: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1732129.916297] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:487823 to 0x800000400:487841 [1732133.500085] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:487694 to 0xbc0000402:487713 [1732137.512211] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:487504 to 0xb00000402:487521 [1732167.632579] Lustre: 21389:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549509877/real 1549509882] req@ffff909e193cbf00 x1624752268786976/t0(0) o400->fir-MDT0002-lwp-OST000b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549509946 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1732167.632602] Lustre: fir-MDT0000-lwp-OST0003: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1732167.632605] Lustre: Skipped 4 previous similar messages [1732167.683133] Lustre: 21389:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1732168.055970] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1732169.056002] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1732169.066259] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [1732191.931625] Lustre: fir-OST0003: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1732191.931828] Lustre: fir-OST0003: deleting orphan objects from 0xbc0000402:487694 to 0xbc0000402:487745 [1732191.950243] Lustre: Skipped 3 previous similar messages [1732191.956878] Lustre: fir-OST0003: deleting orphan objects from 0x0:686747 to 0x0:686785 [1732214.726169] Lustre: 21391:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549509877/real 1549509877] req@ffff909e193c8900 x1624752268786640/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1549509929 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1732214.754372] Lustre: 21391:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1732214.764225] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1732231.778604] Lustre: 21384:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549509877/real 1549509877] req@ffff909e193cc500 x1624752268786720/t0(0) o400->fir-MDT0002-lwp-OST0007@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549509946 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1732231.778606] Lustre: 21380:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549509877/real 1549509877] req@ffff909e193ccb00 x1624752268786656/t0(0) o400->fir-MDT0002-lwp-OST0001@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549509946 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1732231.778610] Lustre: 21380:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [1732231.778616] Lustre: fir-MDT0002-lwp-OST0005: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1732231.778617] Lustre: fir-MDT0000-lwp-OST0007: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1732231.778617] Lustre: Skipped 4 previous similar messages [1732231.778618] Lustre: Skipped 6 previous similar messages [1732233.916151] Lustre: fir-OST000b: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1732233.925290] Lustre: Skipped 7 previous similar messages [1732238.576147] Lustre: fir-OST000b: deleting orphan objects from 0x380000400:487443 to 0x380000400:487457 [1732238.576203] Lustre: fir-OST0009: deleting orphan objects from 0x540000400:487760 to 0x540000400:487777 [1732238.576206] Lustre: fir-OST0007: deleting orphan objects from 0xb00000402:487526 to 0xb00000402:487553 [1732238.576357] Lustre: fir-OST0009: deleting orphan objects from 0x0:686814 to 0x0:686849 [1732238.576459] Lustre: fir-OST0003: deleting orphan objects from 0x0:686747 to 0x0:686817 [1732241.486442] Lustre: fir-OST0005: deleting orphan objects from 0x800000400:487846 to 0x800000400:487873 [1732241.486569] Lustre: fir-OST0001: deleting orphan objects from 0xb40000402:488111 to 0xb40000402:488129 [1732249.776287] Lustre: fir-OST0005: deleting orphan objects from 0x0:686764 to 0x0:686785 [1732270.556898] Lustre: fir-OST000b: deleting orphan objects from 0x0:686941 to 0x0:686977 [1732270.557015] Lustre: fir-OST0007: deleting orphan objects from 0x0:686885 to 0x0:686913 [1732270.557088] Lustre: fir-OST0001: deleting orphan objects from 0x0:687128 to 0x0:687169 [1732288.982178] LustreError: 137-5: fir-OST0002_UUID: not available for connect from 10.9.114.6@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1732288.999637] LustreError: Skipped 2 previous similar messages [1732314.070857] Lustre: fir-OST0009: Client ff20c7b7-4d81-8856-8ae3-3529cc25f5bc (at 10.9.114.6@o2ib4) reconnecting [1732314.081124] Lustre: Skipped 5 previous similar messages [1732636.075954] Lustre: fir-OST0009: Connection restored to (at 10.8.18.24@o2ib6) [1732636.083355] Lustre: Skipped 402 previous similar messages [1733237.037713] Lustre: fir-OST000b: Connection restored to 7936d72a-098c-6dfd-6893-32b658b77868 (at 10.9.103.9@o2ib4) [1733237.048233] Lustre: Skipped 430 previous similar messages [1733842.407843] Lustre: fir-OST0005: Connection restored to (at 10.9.105.24@o2ib4) [1733842.415330] Lustre: Skipped 599 previous similar messages [1734442.840094] Lustre: fir-OST0001: Connection restored to febb809e-97c1-bc5f-76cf-ed320d88831e (at 10.9.107.22@o2ib4) [1734442.850702] Lustre: Skipped 448 previous similar messages [1735048.639811] Lustre: fir-OST0009: Connection restored to ca76035c-fba3-91d6-92f5-1fd97493e3e8 (at 10.9.107.4@o2ib4) [1735048.650337] Lustre: Skipped 475 previous similar messages [1735650.612777] Lustre: fir-OST000b: Connection restored to (at 10.8.17.6@o2ib6) [1735650.620094] Lustre: Skipped 352 previous similar messages [1737398.962832] Lustre: fir-OST0007: haven't heard from client 247006a5-a53b-2198-186e-804d1b86debf (at 10.8.21.19@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff90b5e8ed4800, cur 1549515114 expire 1549514964 last 1549514887 [1737398.984710] Lustre: Skipped 5 previous similar messages [1738805.118993] Lustre: fir-MDT0003-lwp-OST0001: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1738805.135156] Lustre: Skipped 12 previous similar messages [1738831.834735] Lustre: fir-MDT0000-lwp-OST0001: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1738831.850903] Lustre: Skipped 11 previous similar messages [1738838.064050] Lustre: fir-OST0005: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1738838.073187] Lustre: Skipped 1 previous similar message [1738838.078528] Lustre: fir-OST0005: Connection restored to (at 10.0.10.51@o2ib7) [1738838.085931] Lustre: Skipped 185 previous similar messages [1738838.091670] Lustre: fir-OST0005: deleting orphan objects from 0x0:687310 to 0x0:687329 [1738857.227191] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 18 seconds [1738857.237535] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1738858.227217] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1678 seconds [1738858.237731] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [1738859.227245] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [1738859.237587] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1738862.227315] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1738862.237566] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1738866.227422] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1738866.237675] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 8 previous similar messages [1738906.228460] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 37 seconds [1738956.229726] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 87 seconds [1738956.240065] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 23 previous similar messages [1739006.231000] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 15 seconds [1739006.241347] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 23 previous similar messages [1739107.233590] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 39 seconds [1739107.243938] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 47 previous similar messages [1739257.237424] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1739257.247679] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 71 previous similar messages [1739513.243984] LNetError: 21353:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [1739513.254327] LNetError: 21353:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (8): c: 7, oc: 0, rc: 8 [1739513.266425] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1739513.276676] LNet: 21353:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 141 previous similar messages [1739513.286848] Lustre: 21382:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549517222/real 1549517228] req@ffff909e193cd700 x1624752272895728/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1549517229 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1739513.315412] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1739520.244147] LNetError: 21353:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds [1739520.254491] LNetError: 21353:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (15): c: 6, oc: 0, rc: 8 [1739660.322150] Lustre: Failing over fir-OST000b [1739660.326649] Lustre: Skipped 5 previous similar messages [1739660.337361] LustreError: 24209:0:(tgt_grant.c:1079:tgt_grant_discard()) ASSERTION( tgd->tgd_tot_granted >= ted->ted_grant ) failed: fir-OST0005: tot_granted 2114072895 cli e43da944-d239-923f-8f68-10646264727b/ffff90a28977ac00 ted_grant -12582912 [1739660.359265] LustreError: 24209:0:(tgt_grant.c:1079:tgt_grant_discard()) LBUG [1739660.366492] Pid: 24209, comm: umount 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1739660.376143] Call Trace: [1739660.378780] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [1739660.385519] [] lbug_with_loc+0x4c/0xa0 [libcfs] [1739660.391906] [] tgt_grant_discard+0x1dc/0x1e0 [ptlrpc] [1739660.398876] [] ofd_obd_disconnect+0x74/0x220 [ofd] [1739660.405522] [] class_disconnect_export_list+0x1c6/0x680 [obdclass] [1739660.413582] [] class_disconnect_exports+0x115/0x310 [obdclass] [1739660.421279] [] class_cleanup+0x297/0xbd0 [obdclass] [1739660.428031] [] class_process_config+0x65c/0x2830 [obdclass] [1739660.435465] [] class_manual_cleanup+0x1c6/0x710 [obdclass] [1739660.442816] [] server_put_super+0x8de/0xcd0 [obdclass] [1739660.449827] [] generic_shutdown_super+0x6d/0x100 [1739660.456303] [] kill_anon_super+0x12/0x20 [1739660.462077] [] lustre_kill_super+0x32/0x50 [obdclass] [1739660.468989] [] deactivate_locked_super+0x4e/0x70 [1739660.475456] [] deactivate_super+0x46/0x60 [1739660.481314] [] cleanup_mnt+0x3f/0x80 [1739660.486739] [] __cleanup_mnt+0x12/0x20 [1739660.492338] [] task_work_run+0xbb/0xe0 [1739660.497948] [] do_notify_resume+0xa5/0xc0 [1739660.503813] [] int_signal+0x12/0x17 [1739660.509154] [] 0xffffffffffffffff [1739660.514347] Kernel panic - not syncing: LBUG [1739660.518793] CPU: 0 PID: 24209 Comm: umount Kdump: loaded Tainted: G OE ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1739660.530863] Hardware name: Dell Inc. PowerEdge R6415/065PKD, BIOS 1.6.7 10/29/2018 [1739660.538601] Call Trace: [1739660.541232] [] dump_stack+0x19/0x1b [1739660.546543] [] panic+0xe8/0x21f [1739660.551512] [] lbug_with_loc+0x9b/0xa0 [libcfs] [1739660.557903] [] tgt_grant_discard+0x1dc/0x1e0 [ptlrpc] [1739660.564783] [] ofd_obd_disconnect+0x74/0x220 [ofd] [1739660.571411] [] ? libcfs_nid2str_r+0xe2/0x130 [lnet] [1739660.578124] [] class_disconnect_export_list+0x1c6/0x680 [obdclass] [1739660.586143] [] class_disconnect_exports+0x115/0x310 [obdclass] [1739660.593814] [] class_cleanup+0x297/0xbd0 [obdclass] [1739660.600517] [] ? libcfs_debug_msg+0x57/0x80 [libcfs] [1739660.607324] [] ? class_name2dev_nolock+0x46/0xb0 [obdclass] [1739660.614735] [] class_process_config+0x65c/0x2830 [obdclass] [1739660.622131] [] ? libcfs_debug_msg+0x57/0x80 [libcfs] [1739660.628939] [] class_manual_cleanup+0x1c6/0x710 [obdclass] [1739660.636265] [] server_put_super+0x8de/0xcd0 [obdclass] [1739660.643223] [] generic_shutdown_super+0x6d/0x100 [1739660.649661] [] kill_anon_super+0x12/0x20 [1739660.655427] [] lustre_kill_super+0x32/0x50 [obdclass] [1739660.662296] [] deactivate_locked_super+0x4e/0x70 [1739660.668736] [] deactivate_super+0x46/0x60 [1739660.674571] [] cleanup_mnt+0x3f/0x80 [1739660.679968] [] __cleanup_mnt+0x12/0x20 [1739660.685541] [] task_work_run+0xbb/0xe0 [1739660.691115] [] do_notify_resume+0xa5/0xc0 [1739660.696947] [] int_signal+0x12/0x17