[ 19.906111] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0048), lun(0) [ 19.945629] scsi 0:0:54:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 19.954185] scsi 0:0:54:0: SSP: handle(0x0048), sas_addr(0x5000c500954d0481), phy(53), device_name(0x5000c500954d0480) [ 19.964869] scsi 0:0:54:0: enclosure logical id(0x5001636001a06d7d), slot(53) [ 19.972088] scsi 0:0:54:0: enclosure level(0x0000), connector name( C1 ) [ 19.978894] scsi 0:0:54:0: serial_number(ZA1ALF8F0000R8314T49) [ 19.984726] scsi 0:0:54:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.007909] mpt3sas_cm0: detecting: handle(0x0049), sas_address(0x5000c500954de805), phy(54) [ 20.016348] mpt3sas_cm0: REPORT_LUNS: handle(0x0049), retries(0) [ 20.037476] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0049), lun(0) [ 20.078778] scsi 0:0:55:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 20.087327] scsi 0:0:55:0: SSP: handle(0x0049), sas_addr(0x5000c500954de805), phy(54), device_name(0x5000c500954de804) [ 20.098010] scsi 0:0:55:0: enclosure logical id(0x5001636001a06d7d), slot(54) [ 20.105230] scsi 0:0:55:0: enclosure level(0x0000), connector name( C1 ) [ 20.112036] scsi 0:0:55:0: serial_number(ZA1AM5L80000R8317PC6) [ 20.117869] scsi 0:0:55:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.140918] mpt3sas_cm0: detecting: handle(0x004a), sas_address(0x5000c500954e0321), phy(55) [ 20.149360] mpt3sas_cm0: REPORT_LUNS: handle(0x004a), retries(0) [ 20.162962] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004a), lun(0) [ 20.201992] scsi 0:0:56:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 20.210547] scsi 0:0:56:0: SSP: handle(0x004a), sas_addr(0x5000c500954e0321), phy(55), device_name(0x5000c500954e0320) [ 20.221235] scsi 0:0:56:0: enclosure logical id(0x5001636001a06d7d), slot(55) [ 20.228456] scsi 0:0:56:0: enclosure level(0x0000), connector name( C1 ) [ 20.235260] scsi 0:0:56:0: serial_number(ZA1AM1Z30000R8317REB) [ 20.241095] scsi 0:0:56:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.263914] mpt3sas_cm0: detecting: handle(0x004b), sas_address(0x5000c500954d3865), phy(56) [ 20.272351] mpt3sas_cm0: REPORT_LUNS: handle(0x004b), retries(0) [ 20.284977] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004b), lun(0) [ 20.321157] scsi 0:0:57:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 20.329707] scsi 0:0:57:0: SSP: handle(0x004b), sas_addr(0x5000c500954d3865), phy(56), device_name(0x5000c500954d3864) [ 20.340398] scsi 0:0:57:0: enclosure logical id(0x5001636001a06d7d), slot(56) [ 20.347617] scsi 0:0:57:0: enclosure level(0x0000), connector name( C1 ) [ 20.354421] scsi 0:0:57:0: serial_number(ZA1ALTQH0000R8303HV0) [ 20.360257] scsi 0:0:57:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.382916] mpt3sas_cm0: detecting: handle(0x004c), sas_address(0x5000c500954d8791), phy(57) [ 20.391354] mpt3sas_cm0: REPORT_LUNS: handle(0x004c), retries(0) [ 20.403950] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004c), lun(0) [ 20.441964] scsi 0:0:58:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 20.450519] scsi 0:0:58:0: SSP: handle(0x004c), sas_addr(0x5000c500954d8791), phy(57), device_name(0x5000c500954d8790) [ 20.461205] scsi 0:0:58:0: enclosure logical id(0x5001636001a06d7d), slot(57) [ 20.468423] scsi 0:0:58:0: enclosure level(0x0000), connector name( C1 ) [ 20.475230] scsi 0:0:58:0: serial_number(ZA1AM40E0000R8301CKS) [ 20.481062] scsi 0:0:58:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.503923] mpt3sas_cm0: detecting: handle(0x004d), sas_address(0x5000c500954d3d6d), phy(58) [ 20.512363] mpt3sas_cm0: REPORT_LUNS: handle(0x004d), retries(0) [ 20.525017] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004d), lun(0) [ 20.532092] scsi 0:0:59:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 20.540659] scsi 0:0:59:0: SSP: handle(0x004d), sas_addr(0x5000c500954d3d6d), phy(58), device_name(0x5000c500954d3d6c) [ 20.551342] scsi 0:0:59:0: enclosure logical id(0x5001636001a06d7d), slot(58) [ 20.558561] scsi 0:0:59:0: enclosure level(0x0000), connector name( C1 ) [ 20.565367] scsi 0:0:59:0: serial_number(ZA1ALTKA0000R8302V05) [ 20.571202] scsi 0:0:59:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.593924] mpt3sas_cm0: detecting: handle(0x004e), sas_address(0x5000c500954d33c9), phy(59) [ 20.602361] mpt3sas_cm0: REPORT_LUNS: handle(0x004e), retries(0) [ 20.623119] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004e), lun(0) [ 20.659720] scsi 0:0:60:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 20.668279] scsi 0:0:60:0: SSP: handle(0x004e), sas_addr(0x5000c500954d33c9), phy(59), device_name(0x5000c500954d33c8) [ 20.678962] scsi 0:0:60:0: enclosure logical id(0x5001636001a06d7d), slot(59) [ 20.686180] scsi 0:0:60:0: enclosure level(0x0000), connector name( C1 ) [ 20.692988] scsi 0:0:60:0: serial_number(ZA1ALY100000R83028GN) [ 20.698821] scsi 0:0:60:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.723038] mpt3sas_cm0: detecting: handle(0x0050), sas_address(0x5001636001a06d7d), phy(76) [ 20.731495] mpt3sas_cm0: REPORT_LUNS: handle(0x0050), retries(0) [ 20.737690] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0050), lun(0) [ 20.744144] scsi 0:0:61:0: Enclosure QCT JB4602 SIM 0 1500 PQ: 0 ANSI: 6 [ 20.752551] scsi 0:0:61:0: set ignore_delay_remove for handle(0x0050) [ 20.758991] scsi 0:0:61:0: SES: handle(0x0050), sas_addr(0x5001636001a06d7d), phy(76), device_name(0x5001636001a06d7d) [ 20.769678] scsi 0:0:61:0: enclosure logical id(0x5001636001a06d7d), slot(60) [ 20.776896] scsi 0:0:61:0: enclosure level(0x0000), connector name( C1 ) [ 20.783702] scsi 0:0:61:0: serial_number(EO0153300063 ) [ 20.789189] scsi 0:0:61:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.814018] mpt3sas_cm0: expander_add: handle(0x0051), parent(0x0001), sas_addr(0x500163600190cd7f), phys(79) [ 20.834622] mpt3sas_cm0: detecting: handle(0x0053), sas_address(0x5000c50095430dd1), phy(0) [ 20.842971] mpt3sas_cm0: REPORT_LUNS: handle(0x0053), retries(0) [ 20.859616] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0053), lun(0) [ 20.900092] scsi 0:0:62:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 20.908646] scsi 0:0:62:0: SSP: handle(0x0053), sas_addr(0x5000c50095430dd1), phy(0), device_name(0x5000c50095430dd0) [ 20.919243] scsi 0:0:62:0: enclosure logical id(0x500163600190cd7d), slot(0) [ 20.926375] scsi 0:0:62:0: enclosure level(0x0000), connector name( C3 ) [ 20.933181] scsi 0:0:62:0: serial_number(ZA1AGHL00000C8316HQ4) [ 20.939015] scsi 0:0:62:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.961931] mpt3sas_cm0: detecting: handle(0x0054), sas_address(0x5000c50095432245), phy(1) [ 20.970280] mpt3sas_cm0: REPORT_LUNS: handle(0x0054), retries(0) [ 20.990719] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0054), lun(0) [ 21.026365] scsi 0:0:63:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.034928] scsi 0:0:63:0: SSP: handle(0x0054), sas_addr(0x5000c50095432245), phy(1), device_name(0x5000c50095432244) [ 21.045528] scsi 0:0:63:0: enclosure logical id(0x500163600190cd7d), slot(1) [ 21.052661] scsi 0:0:63:0: enclosure level(0x0000), connector name( C3 ) [ 21.059472] scsi 0:0:63:0: serial_number(ZA1AGN060000R828A07T) [ 21.065311] scsi 0:0:63:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.090935] mpt3sas_cm0: detecting: handle(0x0055), sas_address(0x5000c5009569935d), phy(2) [ 21.099300] mpt3sas_cm0: REPORT_LUNS: handle(0x0055), retries(0) [ 21.112823] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0055), lun(0) [ 21.150461] scsi 0:0:64:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.159019] scsi 0:0:64:0: SSP: handle(0x0055), sas_addr(0x5000c5009569935d), phy(2), device_name(0x5000c5009569935c) [ 21.169622] scsi 0:0:64:0: enclosure logical id(0x500163600190cd7d), slot(2) [ 21.176754] scsi 0:0:64:0: enclosure level(0x0000), connector name( C3 ) [ 21.183561] scsi 0:0:64:0: serial_number(ZA1AZL5Y0000R832DW4F) [ 21.189393] scsi 0:0:64:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.211942] mpt3sas_cm0: detecting: handle(0x0056), sas_address(0x5000c500a6102e85), phy(3) [ 21.220295] mpt3sas_cm0: REPORT_LUNS: handle(0x0056), retries(0) [ 21.238600] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0056), lun(0) [ 21.277035] scsi 0:0:65:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.285595] scsi 0:0:65:0: SSP: handle(0x0056), sas_addr(0x5000c500a6102e85), phy(3), device_name(0x5000c500a6102e84) [ 21.296193] scsi 0:0:65:0: enclosure logical id(0x500163600190cd7d), slot(3) [ 21.303326] scsi 0:0:65:0: enclosure level(0x0000), connector name( C3 ) [ 21.310130] scsi 0:0:65:0: serial_number(ZA1B72S80000R8332SVA) [ 21.315965] scsi 0:0:65:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.338944] mpt3sas_cm0: detecting: handle(0x0057), sas_address(0x5000c50095431c1d), phy(4) [ 21.347298] mpt3sas_cm0: REPORT_LUNS: handle(0x0057), retries(0) [ 21.365335] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0057), lun(0) [ 21.404883] scsi 0:0:66:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.413437] scsi 0:0:66:0: SSP: handle(0x0057), sas_addr(0x5000c50095431c1d), phy(4), device_name(0x0000000000000000) [ 21.424037] scsi 0:0:66:0: enclosure logical id(0x500163600190cd7d), slot(4) [ 21.431171] scsi 0:0:66:0: enclosure level(0x0000), connector name( C3 ) [ 21.437978] scsi 0:0:66:0: serial_number(ZA1AGN520000C8316G32) [ 21.443811] scsi 0:0:66:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.466946] mpt3sas_cm0: detecting: handle(0x0058), sas_address(0x5000c500957faccd), phy(5) [ 21.475299] mpt3sas_cm0: REPORT_LUNS: handle(0x0058), retries(0) [ 21.486948] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0058), lun(0) [ 21.494029] scsi 0:0:67:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.502601] scsi 0:0:67:0: SSP: handle(0x0058), sas_addr(0x5000c500957faccd), phy(5), device_name(0x5000c500957faccc) [ 21.513204] scsi 0:0:67:0: enclosure logical id(0x500163600190cd7d), slot(5) [ 21.520337] scsi 0:0:67:0: enclosure level(0x0000), connector name( C3 ) [ 21.527141] scsi 0:0:67:0: serial_number(ZA1B4F0J0000R833TXB0) [ 21.532976] scsi 0:0:67:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.555948] mpt3sas_cm0: detecting: handle(0x0059), sas_address(0x5000c5009543265d), phy(6) [ 21.564305] mpt3sas_cm0: REPORT_LUNS: handle(0x0059), retries(0) [ 21.577350] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0059), lun(0) [ 21.614469] scsi 0:0:68:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.623026] scsi 0:0:68:0: SSP: handle(0x0059), sas_addr(0x5000c5009543265d), phy(6), device_name(0x5000c5009543265c) [ 21.633630] scsi 0:0:68:0: enclosure logical id(0x500163600190cd7d), slot(6) [ 21.640762] scsi 0:0:68:0: enclosure level(0x0000), connector name( C3 ) [ 21.647568] scsi 0:0:68:0: serial_number(ZA1AGGRH0000R829CU3W) [ 21.653401] scsi 0:0:68:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.675953] mpt3sas_cm0: detecting: handle(0x005a), sas_address(0x5000c50095432469), phy(7) [ 21.684303] mpt3sas_cm0: REPORT_LUNS: handle(0x005a), retries(0) [ 21.704915] mpt3sas_cm0: TEST_UNIT_READY: handle(0x005a), lun(0) [ 21.740485] scsi 0:0:69:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.749039] scsi 0:0:69:0: SSP: handle(0x005a), sas_addr(0x5000c50095432469), phy(7), device_name(0x5000c50095432468) [ 21.759639] scsi 0:0:69:0: enclosure logical id(0x500163600190cd7d), slot(7) [ 21.766772] scsi 0:0:69:0: enclosure level(0x0000), connector name( C3 ) [ 21.773577] scsi 0:0:69:0: serial_number(ZA1AGMY10000C8316H8M) [ 21.779411] scsi 0:0:69:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.801956] mpt3sas_cm0: detecting: handle(0x0063), sas_address(0x5000c5009543205d), phy(8) [ 21.810312] mpt3sas_cm0: REPORT_LUNS: handle(0x0063), retries(0) [ 21.825385] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0063), lun(0) [ 21.862558] scsi 0:0:70:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.871112] scsi 0:0:70:0: SSP: handle(0x0063), sas_addr(0x5000c5009543205d), phy(8), device_name(0x5000c5009543205c) [ 21.881711] scsi 0:0:70:0: enclosure logical id(0x500163600190cd7d), slot(8) [ 21.888843] scsi 0:0:70:0: enclosure level(0x0000), connector name( C3 ) [ 21.895650] scsi 0:0:70:0: serial_number(ZA1AGN210000R82892FE) [ 21.901484] scsi 0:0:70:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.924960] mpt3sas_cm0: detecting: handle(0x0064), sas_address(0x5000c50095432225), phy(9) [ 21.933310] mpt3sas_cm0: REPORT_LUNS: handle(0x0064), retries(0) [ 21.950734] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0064), lun(0) [ 21.957834] scsi 0:0:71:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 21.998564] scsi 0:0:71:0: SSP: handle(0x0064), sas_addr(0x5000c50095432225), phy(9), device_name(0x5000c50095432224) [ 22.009167] scsi 0:0:71:0: enclosure logical id(0x500163600190cd7d), slot(9) [ 22.016299] scsi 0:0:71:0: enclosure level(0x0000), connector name( C3 ) [ 22.023107] scsi 0:0:71:0: serial_number(ZA1AGN0E0000R82893RN) [ 22.028941] scsi 0:0:71:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.051964] mpt3sas_cm0: detecting: handle(0x0065), sas_address(0x5000c50095431219), phy(10) [ 22.060402] mpt3sas_cm0: REPORT_LUNS: handle(0x0065), retries(0) [ 22.079462] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0065), lun(0) [ 22.116711] scsi 0:0:72:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 22.125270] scsi 0:0:72:0: SSP: handle(0x0065), sas_addr(0x5000c50095431219), phy(10), device_name(0x5000c50095431218) [ 22.135956] scsi 0:0:72:0: enclosure logical id(0x500163600190cd7d), slot(10) [ 22.143174] scsi 0:0:72:0: enclosure level(0x0000), connector name( C3 ) [ 22.149982] scsi 0:0:72:0: serial_number(ZA1AGHEZ0000R8289ZB0) [ 22.155815] scsi 0:0:72:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.178962] mpt3sas_cm0: detecting: handle(0x0066), sas_address(0x5000c5009543250d), phy(11) [ 22.187399] mpt3sas_cm0: REPORT_LUNS: handle(0x0066), retries(0) [ 22.208029] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0066), lun(0) [ 22.240935] scsi 0:0:73:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 22.249496] scsi 0:0:73:0: SSP: handle(0x0066), sas_addr(0x5000c5009543250d), phy(11), device_name(0x5000c5009543250c) [ 22.260178] scsi 0:0:73:0: enclosure logical id(0x500163600190cd7d), slot(11) [ 22.267398] scsi 0:0:73:0: enclosure level(0x0000), connector name( C3 ) [ 22.274204] scsi 0:0:73:0: serial_number(ZA1AGMX90000C8316H8U) [ 22.280037] scsi 0:0:73:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.302966] mpt3sas_cm0: detecting: handle(0x0067), sas_address(0x5000c500954cf2a9), phy(12) [ 22.311404] mpt3sas_cm0: REPORT_LUNS: handle(0x0067), retries(0) [ 22.325166] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0067), lun(0) [ 22.363590] scsi 0:0:74:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 22.372142] scsi 0:0:74:0: SSP: handle(0x0067), sas_addr(0x5000c500954cf2a9), phy(12), device_name(0x5000c500954cf2a8) [ 22.382831] scsi 0:0:74:0: enclosure logical id(0x500163600190cd7d), slot(12) [ 22.390051] scsi 0:0:74:0: enclosure level(0x0000), connector name( C3 ) [ 22.396859] scsi 0:0:74:0: serial_number(ZA1ALFL80000R8314RU0) [ 22.402691] scsi 0:0:74:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.425969] mpt3sas_cm0: detecting: handle(0x0068), sas_address(0x5000c500954d4119), phy(13) [ 22.434405] mpt3sas_cm0: REPORT_LUNS: handle(0x0068), retries(0) [ 22.445439] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0068), lun(0) [ 22.486031] scsi 0:0:75:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 22.494588] scsi 0:0:75:0: SSP: handle(0x0068), sas_addr(0x5000c500954d4119), phy(13), device_name(0x5000c500954d4118) [ 22.505269] scsi 0:0:75:0: enclosure logical id(0x500163600190cd7d), slot(13) [ 22.512489] scsi 0:0:75:0: enclosure level(0x0000), connector name( C3 ) [ 22.519294] scsi 0:0:75:0: serial_number(ZA1ALXPR0000R8302J8K) [ 22.525129] scsi 0:0:75:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.547972] mpt3sas_cm0: detecting: handle(0x0069), sas_address(0x5000c500954cb5e1), phy(14) [ 22.556413] mpt3sas_cm0: REPORT_LUNS: handle(0x0069), retries(0) [ 22.562983] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0069), lun(0) [ 23.123593] scsi 0:0:76:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 23.132306] scsi 0:0:76:0: SSP: handle(0x0069), sas_addr(0x5000c500954cb5e1), phy(14), device_name(0x5000c500954cb5e0) [ 23.142988] scsi 0:0:76:0: enclosure logical id(0x500163600190cd7d), slot(14) [ 23.150209] scsi 0:0:76:0: enclosure level(0x0000), connector name( C3 ) [ 23.157014] scsi 0:0:76:0: serial_number(ZA1ALH7V0000C8326GSW) [ 23.162847] scsi 0:0:76:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.584032] mpt3sas_cm0: detecting: handle(0x006a), sas_address(0x5000c500954df389), phy(15) [ 23.592485] mpt3sas_cm0: REPORT_LUNS: handle(0x006a), retries(0) [ 23.599015] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006a), lun(0) [ 23.606078] scsi 0:0:77:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 23.614651] scsi 0:0:77:0: SSP: handle(0x006a), sas_addr(0x5000c500954df389), phy(15), device_name(0x5000c500954df388) [ 23.625337] scsi 0:0:77:0: enclosure logical id(0x500163600190cd7d), slot(15) [ 23.632557] scsi 0:0:77:0: enclosure level(0x0000), connector name( C3 ) [ 23.639360] scsi 0:0:77:0: serial_number(ZA1AM2550000R8317NB5) [ 23.645195] scsi 0:0:77:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.667998] mpt3sas_cm0: detecting: handle(0x0073), sas_address(0x5000c500954d8f5d), phy(16) [ 23.676436] mpt3sas_cm0: REPORT_LUNS: handle(0x0073), retries(0) [ 23.695057] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0073), lun(0) [ 23.733606] scsi 0:0:78:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 23.742158] scsi 0:0:78:0: SSP: handle(0x0073), sas_addr(0x5000c500954d8f5d), phy(16), device_name(0x5000c500954d8f5c) [ 23.752844] scsi 0:0:78:0: enclosure logical id(0x500163600190cd7d), slot(16) [ 23.760063] scsi 0:0:78:0: enclosure level(0x0000), connector name( C3 ) [ 23.766871] scsi 0:0:78:0: serial_number(ZA1A3EEM0000R8301QYF) [ 23.772703] scsi 0:0:78:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.796004] mpt3sas_cm0: detecting: handle(0x0074), sas_address(0x5000c500954ccd99), phy(17) [ 23.804446] mpt3sas_cm0: REPORT_LUNS: handle(0x0074), retries(0) [ 23.819906] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0074), lun(0) [ 23.858649] scsi 0:0:79:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 23.867207] scsi 0:0:79:0: SSP: handle(0x0074), sas_addr(0x5000c500954ccd99), phy(17), device_name(0x5000c500954ccd98) [ 23.877890] scsi 0:0:79:0: enclosure logical id(0x500163600190cd7d), slot(17) [ 23.885112] scsi 0:0:79:0: enclosure level(0x0000), connector name( C3 ) [ 23.891916] scsi 0:0:79:0: serial_number(ZA1ALL1N0000R8301RWS) [ 23.897749] scsi 0:0:79:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.921000] mpt3sas_cm0: detecting: handle(0x0075), sas_address(0x5000c500954d2dbd), phy(18) [ 23.929439] mpt3sas_cm0: REPORT_LUNS: handle(0x0075), retries(0) [ 23.942221] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0075), lun(0) [ 23.977983] scsi 0:0:80:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 23.986542] scsi 0:0:80:0: SSP: handle(0x0075), sas_addr(0x5000c500954d2dbd), phy(18), device_name(0x5000c500954d2dbc) [ 23.997226] scsi 0:0:80:0: enclosure logical id(0x500163600190cd7d), slot(18) [ 24.004444] scsi 0:0:80:0: enclosure level(0x0000), connector name( C3 ) [ 24.011251] scsi 0:0:80:0: serial_number(ZA1ALV6M0000R8314SFK) [ 24.017083] scsi 0:0:80:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.040007] mpt3sas_cm0: detecting: handle(0x0076), sas_address(0x5000c500954cc1b5), phy(19) [ 24.048443] mpt3sas_cm0: REPORT_LUNS: handle(0x0076), retries(0) [ 24.066295] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0076), lun(0) [ 24.103588] scsi 0:0:81:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.112141] scsi 0:0:81:0: SSP: handle(0x0076), sas_addr(0x5000c500954cc1b5), phy(19), device_name(0x5000c500954cc1b4) [ 24.122827] scsi 0:0:81:0: enclosure logical id(0x500163600190cd7d), slot(19) [ 24.130046] scsi 0:0:81:0: enclosure level(0x0000), connector name( C3 ) [ 24.136854] scsi 0:0:81:0: serial_number(ZA1ALGWT0000C8325X34) [ 24.142687] scsi 0:0:81:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.166012] mpt3sas_cm0: detecting: handle(0x0077), sas_address(0x5000c500954310fd), phy(20) [ 24.174452] mpt3sas_cm0: REPORT_LUNS: handle(0x0077), retries(0) [ 24.193285] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0077), lun(0) [ 24.232102] scsi 0:0:82:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.240660] scsi 0:0:82:0: SSP: handle(0x0077), sas_addr(0x5000c500954310fd), phy(20), device_name(0x5000c500954310fc) [ 24.251349] scsi 0:0:82:0: enclosure logical id(0x500163600190cd7d), slot(20) [ 24.258567] scsi 0:0:82:0: enclosure level(0x0000), connector name( C3 ) [ 24.265373] scsi 0:0:82:0: serial_number(ZA1AGHGR0000C8316FM1) [ 24.271205] scsi 0:0:82:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.294012] mpt3sas_cm0: detecting: handle(0x0078), sas_address(0x5000c500954cc171), phy(21) [ 24.302453] mpt3sas_cm0: REPORT_LUNS: handle(0x0078), retries(0) [ 24.318891] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0078), lun(0) [ 24.356095] scsi 0:0:83:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.364654] scsi 0:0:83:0: SSP: handle(0x0078), sas_addr(0x5000c500954cc171), phy(21), device_name(0x5000c500954cc170) [ 24.375338] scsi 0:0:83:0: enclosure logical id(0x500163600190cd7d), slot(21) [ 24.382556] scsi 0:0:83:0: enclosure level(0x0000), connector name( C3 ) [ 24.389361] scsi 0:0:83:0: serial_number(ZA1ALGXA0000C8325WZ8) [ 24.395194] scsi 0:0:83:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.418606] mpt3sas_cm0: detecting: handle(0x0079), sas_address(0x5000c500954ddfe1), phy(22) [ 24.427040] mpt3sas_cm0: REPORT_LUNS: handle(0x0079), retries(0) [ 24.445188] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0079), lun(0) [ 24.452289] scsi 0:0:84:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.460859] scsi 0:0:84:0: SSP: handle(0x0079), sas_addr(0x5000c500954ddfe1), phy(22), device_name(0x5000c500954ddfe0) [ 24.471547] scsi 0:0:84:0: enclosure logical id(0x500163600190cd7d), slot(22) [ 24.478768] scsi 0:0:84:0: enclosure level(0x0000), connector name( C3 ) [ 24.485573] scsi 0:0:84:0: serial_number(ZA1AM5R10000R8317NT7) [ 24.491406] scsi 0:0:84:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.514018] mpt3sas_cm0: detecting: handle(0x007a), sas_address(0x5000c500954cb455), phy(23) [ 24.522453] mpt3sas_cm0: REPORT_LUNS: handle(0x007a), retries(0) [ 24.533863] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007a), lun(0) [ 24.573003] scsi 0:0:85:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.581561] scsi 0:0:85:0: SSP: handle(0x007a), sas_addr(0x5000c500954cb455), phy(23), device_name(0x5000c500954cb454) [ 24.592244] scsi 0:0:85:0: enclosure logical id(0x500163600190cd7d), slot(23) [ 24.599462] scsi 0:0:85:0: enclosure level(0x0000), connector name( C3 ) [ 24.606270] scsi 0:0:85:0: serial_number(ZA1ALHA50000C8325WZL) [ 24.612103] scsi 0:0:85:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.635018] mpt3sas_cm0: detecting: handle(0x0083), sas_address(0x5000c500a6104dbd), phy(24) [ 24.643454] mpt3sas_cm0: REPORT_LUNS: handle(0x0083), retries(0) [ 24.660251] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0083), lun(0) [ 24.667348] scsi 0:0:86:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.675927] scsi 0:0:86:0: SSP: handle(0x0083), sas_addr(0x5000c500a6104dbd), phy(24), device_name(0x5000c500a6104dbc) [ 24.686608] scsi 0:0:86:0: enclosure logical id(0x500163600190cd7d), slot(24) [ 24.693829] scsi 0:0:86:0: enclosure level(0x0000), connector name( C3 ) [ 24.700633] scsi 0:0:86:0: serial_number(ZA1B729S0000R824UQU0) [ 24.706468] scsi 0:0:86:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.729021] mpt3sas_cm0: detecting: handle(0x0084), sas_address(0x5000c500954d0c09), phy(25) [ 24.737464] mpt3sas_cm0: REPORT_LUNS: handle(0x0084), retries(0) [ 24.758792] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0084), lun(0) [ 24.794503] scsi 0:0:87:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.803054] scsi 0:0:87:0: SSP: handle(0x0084), sas_addr(0x5000c500954d0c09), phy(25), device_name(0x5000c500954d0c08) [ 24.813743] scsi 0:0:87:0: enclosure logical id(0x500163600190cd7d), slot(25) [ 24.820963] scsi 0:0:87:0: enclosure level(0x0000), connector name( C3 ) [ 24.827771] scsi 0:0:87:0: serial_number(ZA1AFAX70000R8314QSP) [ 24.833601] scsi 0:0:87:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.857025] mpt3sas_cm0: detecting: handle(0x0085), sas_address(0x5000c500954d91a1), phy(26) [ 24.865465] mpt3sas_cm0: REPORT_LUNS: handle(0x0085), retries(0) [ 24.876251] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0085), lun(0) [ 24.883325] scsi 0:0:88:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 24.891895] scsi 0:0:88:0: SSP: handle(0x0085), sas_addr(0x5000c500954d91a1), phy(26), device_name(0x5000c500954d91a0) [ 24.902579] scsi 0:0:88:0: enclosure logical id(0x500163600190cd7d), slot(26) [ 24.909799] scsi 0:0:88:0: enclosure level(0x0000), connector name( C3 ) [ 24.916604] scsi 0:0:88:0: serial_number(ZA1A7W9H0000R8302K3E) [ 24.922438] scsi 0:0:88:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.945022] mpt3sas_cm0: detecting: handle(0x0086), sas_address(0x5000c500a610243d), phy(27) [ 24.953461] mpt3sas_cm0: REPORT_LUNS: handle(0x0086), retries(0) [ 24.964362] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0086), lun(0) [ 25.004938] scsi 0:0:89:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.013489] scsi 0:0:89:0: SSP: handle(0x0086), sas_addr(0x5000c500a610243d), phy(27), device_name(0x5000c500a610243c) [ 25.024175] scsi 0:0:89:0: enclosure logical id(0x500163600190cd7d), slot(27) [ 25.031395] scsi 0:0:89:0: enclosure level(0x0000), connector name( C3 ) [ 25.038204] scsi 0:0:89:0: serial_number(ZA1B72Z10000R833TUKN) [ 25.044045] scsi 0:0:89:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.073027] mpt3sas_cm0: detecting: handle(0x0087), sas_address(0x5000c500a6102eb1), phy(28) [ 25.081477] mpt3sas_cm0: REPORT_LUNS: handle(0x0087), retries(0) [ 25.102527] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0087), lun(0) [ 25.109622] scsi 0:0:90:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.118197] scsi 0:0:90:0: SSP: handle(0x0087), sas_addr(0x5000c500a6102eb1), phy(28), device_name(0x5000c500a6102eb0) [ 25.128881] scsi 0:0:90:0: enclosure logical id(0x500163600190cd7d), slot(28) [ 25.136101] scsi 0:0:90:0: enclosure level(0x0000), connector name( C3 ) [ 25.142906] scsi 0:0:90:0: serial_number(ZA1B72R60000R83323UR) [ 25.148739] scsi 0:0:90:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.172033] mpt3sas_cm0: detecting: handle(0x0088), sas_address(0x5000c500a6102439), phy(29) [ 25.180472] mpt3sas_cm0: REPORT_LUNS: handle(0x0088), retries(0) [ 25.187020] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0088), lun(0) [ 25.229240] scsi 0:0:91:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.237802] scsi 0:0:91:0: SSP: handle(0x0088), sas_addr(0x5000c500a6102439), phy(29), device_name(0x5000c500a6102438) [ 25.248484] scsi 0:0:91:0: enclosure logical id(0x500163600190cd7d), slot(29) [ 25.255703] scsi 0:0:91:0: enclosure level(0x0000), connector name( C3 ) [ 25.262510] scsi 0:0:91:0: serial_number(ZA1B72ZL0000R833UX7J) [ 25.268342] scsi 0:0:91:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.291039] mpt3sas_cm0: detecting: handle(0x0089), sas_address(0x5000c500954d9981), phy(30) [ 25.299479] mpt3sas_cm0: REPORT_LUNS: handle(0x0089), retries(0) [ 25.318538] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0089), lun(0) [ 25.325629] scsi 0:0:92:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.334199] scsi 0:0:92:0: SSP: handle(0x0089), sas_addr(0x5000c500954d9981), phy(30), device_name(0x5000c500954d9980) [ 25.344887] scsi 0:0:92:0: enclosure logical id(0x500163600190cd7d), slot(30) [ 25.352105] scsi 0:0:92:0: enclosure level(0x0000), connector name( C3 ) [ 25.358912] scsi 0:0:92:0: serial_number(ZA1AM3Q20000R8302N04) [ 25.364745] scsi 0:0:92:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.388036] mpt3sas_cm0: detecting: handle(0x008a), sas_address(0x5000c500954deab9), phy(31) [ 25.396477] mpt3sas_cm0: REPORT_LUNS: handle(0x008a), retries(0) [ 25.415235] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008a), lun(0) [ 25.453805] scsi 0:0:93:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.462364] scsi 0:0:93:0: SSP: handle(0x008a), sas_addr(0x5000c500954deab9), phy(31), device_name(0x5000c500954deab8) [ 25.473053] scsi 0:0:93:0: enclosure logical id(0x500163600190cd7d), slot(31) [ 25.480271] scsi 0:0:93:0: enclosure level(0x0000), connector name( C3 ) [ 25.487078] scsi 0:0:93:0: serial_number(ZA1AJVZZ0000R8317PEX) [ 25.492911] scsi 0:0:93:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.516039] mpt3sas_cm0: detecting: handle(0x0093), sas_address(0x5000c500954cc8d5), phy(32) [ 25.524479] mpt3sas_cm0: REPORT_LUNS: handle(0x0093), retries(0) [ 25.544261] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0093), lun(0) [ 25.551348] scsi 0:0:94:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.559942] scsi 0:0:94:0: SSP: handle(0x0093), sas_addr(0x5000c500954cc8d5), phy(32), device_name(0x5000c500954cc8d4) [ 25.570625] scsi 0:0:94:0: enclosure logical id(0x500163600190cd7d), slot(32) [ 25.577844] scsi 0:0:94:0: enclosure level(0x0000), connector name( C3 ) [ 25.584652] scsi 0:0:94:0: serial_number(ZA1ALGNR0000R8302KGJ) [ 25.590484] scsi 0:0:94:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.613042] mpt3sas_cm0: detecting: handle(0x0094), sas_address(0x5000c500a6102de1), phy(33) [ 25.621479] mpt3sas_cm0: REPORT_LUNS: handle(0x0094), retries(0) [ 25.643008] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0094), lun(0) [ 25.650112] scsi 0:0:95:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.658679] scsi 0:0:95:0: SSP: handle(0x0094), sas_addr(0x5000c500a6102de1), phy(33), device_name(0x5000c500a6102de0) [ 25.669368] scsi 0:0:95:0: enclosure logical id(0x500163600190cd7d), slot(33) [ 25.676586] scsi 0:0:95:0: enclosure level(0x0000), connector name( C3 ) [ 25.683390] scsi 0:0:95:0: serial_number(ZA1B72S70000R833V13M) [ 25.689225] scsi 0:0:95:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.712050] mpt3sas_cm0: detecting: handle(0x0095), sas_address(0x5000c500954cc655), phy(34) [ 25.720488] mpt3sas_cm0: REPORT_LUNS: handle(0x0095), retries(0) [ 25.737047] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0095), lun(0) [ 25.772702] scsi 0:0:96:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.781259] scsi 0:0:96:0: SSP: handle(0x0095), sas_addr(0x5000c500954cc655), phy(34), device_name(0x5000c500954cc654) [ 25.791944] scsi 0:0:96:0: enclosure logical id(0x500163600190cd7d), slot(34) [ 25.799162] scsi 0:0:96:0: enclosure level(0x0000), connector name( C3 ) [ 25.805968] scsi 0:0:96:0: serial_number(ZA1ALGS20000C8327479) [ 25.811801] scsi 0:0:96:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.835048] mpt3sas_cm0: detecting: handle(0x0096), sas_address(0x5000c500954d38c9), phy(35) [ 25.843503] mpt3sas_cm0: REPORT_LUNS: handle(0x0096), retries(0) [ 25.856881] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0096), lun(0) [ 25.893686] scsi 0:0:97:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.902241] scsi 0:0:97:0: SSP: handle(0x0096), sas_addr(0x5000c500954d38c9), phy(35), device_name(0x5000c500954d38c8) [ 25.912925] scsi 0:0:97:0: enclosure logical id(0x500163600190cd7d), slot(35) [ 25.920144] scsi 0:0:97:0: enclosure level(0x0000), connector name( C3 ) [ 25.926949] scsi 0:0:97:0: serial_number(ZA1ALXWM0000R83026JR) [ 25.932782] scsi 0:0:97:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.956054] mpt3sas_cm0: detecting: handle(0x0097), sas_address(0x5000c500954d94d1), phy(36) [ 25.964491] mpt3sas_cm0: REPORT_LUNS: handle(0x0097), retries(0) [ 25.980803] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0097), lun(0) [ 25.987888] scsi 0:0:98:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 25.996456] scsi 0:0:98:0: SSP: handle(0x0097), sas_addr(0x5000c500954d94d1), phy(36), device_name(0x5000c500954d94d0) [ 26.007142] scsi 0:0:98:0: enclosure logical id(0x500163600190cd7d), slot(36) [ 26.014362] scsi 0:0:98:0: enclosure level(0x0000), connector name( C3 ) [ 26.021167] scsi 0:0:98:0: serial_number(ZA1AM3VG0000R8313JPX) [ 26.027000] scsi 0:0:98:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.050050] mpt3sas_cm0: detecting: handle(0x0098), sas_address(0x5000c500a6105289), phy(37) [ 26.058490] mpt3sas_cm0: REPORT_LUNS: handle(0x0098), retries(0) [ 26.069839] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0098), lun(0) [ 26.102961] scsi 0:0:99:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.111521] scsi 0:0:99:0: SSP: handle(0x0098), sas_addr(0x5000c500a6105289), phy(37), device_name(0x5000c500a6105288) [ 26.122204] scsi 0:0:99:0: enclosure logical id(0x500163600190cd7d), slot(37) [ 26.129422] scsi 0:0:99:0: enclosure level(0x0000), connector name( C3 ) [ 26.136230] scsi 0:0:99:0: serial_number(ZA1B727K0000R8372HYT) [ 26.142063] scsi 0:0:99:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.166057] mpt3sas_cm0: detecting: handle(0x0099), sas_address(0x5000c500a6103fd9), phy(38) [ 26.174499] mpt3sas_cm0: REPORT_LUNS: handle(0x0099), retries(0) [ 26.195301] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0099), lun(0) [ 26.236963] scsi 0:0:100:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.245627] scsi 0:0:100:0: SSP: handle(0x0099), sas_addr(0x5000c500a6103fd9), phy(38), device_name(0x5000c500a6103fd8) [ 26.256403] scsi 0:0:100:0: enclosure logical id(0x500163600190cd7d), slot(38) [ 26.263707] scsi 0:0:100:0: enclosure level(0x0000), connector name( C3 ) [ 26.270600] scsi 0:0:100:0: serial_number(ZA1B72HS0000R8331QSS) [ 26.276520] scsi 0:0:100:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.298059] mpt3sas_cm0: detecting: handle(0x009a), sas_address(0x5000c500a6105755), phy(39) [ 26.306495] mpt3sas_cm0: REPORT_LUNS: handle(0x009a), retries(0) [ 26.324548] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009a), lun(0) [ 26.363568] scsi 0:0:101:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.372207] scsi 0:0:101:0: SSP: handle(0x009a), sas_addr(0x5000c500a6105755), phy(39), device_name(0x5000c500a6105754) [ 26.382983] scsi 0:0:101:0: enclosure logical id(0x500163600190cd7d), slot(39) [ 26.390290] scsi 0:0:101:0: enclosure level(0x0000), connector name( C3 ) [ 26.397181] scsi 0:0:101:0: serial_number(ZA1B72670000R833UYPW) [ 26.403102] scsi 0:0:101:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.426061] mpt3sas_cm0: detecting: handle(0x00a3), sas_address(0x5000c500956c75a1), phy(40) [ 26.434494] mpt3sas_cm0: REPORT_LUNS: handle(0x00a3), retries(0) [ 26.446893] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a3), lun(0) [ 26.485104] scsi 0:0:102:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.493750] scsi 0:0:102:0: SSP: handle(0x00a3), sas_addr(0x5000c500956c75a1), phy(40), device_name(0x5000c500956c75a0) [ 26.504518] scsi 0:0:102:0: enclosure logical id(0x500163600190cd7d), slot(40) [ 26.511825] scsi 0:0:102:0: enclosure level(0x0000), connector name( C3 ) [ 26.518715] scsi 0:0:102:0: serial_number(ZA1B0V530000R8332QYC) [ 26.524638] scsi 0:0:102:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.548066] mpt3sas_cm0: detecting: handle(0x00a4), sas_address(0x5000c500a61033e1), phy(41) [ 26.556511] mpt3sas_cm0: REPORT_LUNS: handle(0x00a4), retries(0) [ 26.574702] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a4), lun(0) [ 26.612256] scsi 0:0:103:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.620902] scsi 0:0:103:0: SSP: handle(0x00a4), sas_addr(0x5000c500a61033e1), phy(41), device_name(0x5000c500a61033e0) [ 26.631671] scsi 0:0:103:0: enclosure logical id(0x500163600190cd7d), slot(41) [ 26.638977] scsi 0:0:103:0: enclosure level(0x0000), connector name( C3 ) [ 26.645870] scsi 0:0:103:0: serial_number(ZA1B737J0000R8332RQ1) [ 26.651790] scsi 0:0:103:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.675068] mpt3sas_cm0: detecting: handle(0x00a5), sas_address(0x5000c500a6104d25), phy(42) [ 26.683505] mpt3sas_cm0: REPORT_LUNS: handle(0x00a5), retries(0) [ 26.695642] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a5), lun(0) [ 26.730002] scsi 0:0:104:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.738646] scsi 0:0:104:0: SSP: handle(0x00a5), sas_addr(0x5000c500a6104d25), phy(42), device_name(0x5000c500a6104d24) [ 26.749418] scsi 0:0:104:0: enclosure logical id(0x500163600190cd7d), slot(42) [ 26.756725] scsi 0:0:104:0: enclosure level(0x0000), connector name( C3 ) [ 26.763618] scsi 0:0:104:0: serial_number(ZA1B72AS0000R833UYB9) [ 26.769539] scsi 0:0:104:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.793073] mpt3sas_cm0: detecting: handle(0x00a6), sas_address(0x5000c500a6102115), phy(43) [ 26.801513] mpt3sas_cm0: REPORT_LUNS: handle(0x00a6), retries(0) [ 26.812688] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a6), lun(0) [ 26.853856] scsi 0:0:105:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.862501] scsi 0:0:105:0: SSP: handle(0x00a6), sas_addr(0x5000c500a6102115), phy(43), device_name(0x5000c500a6102114) [ 26.873271] scsi 0:0:105:0: enclosure logical id(0x500163600190cd7d), slot(43) [ 26.880575] scsi 0:0:105:0: enclosure level(0x0000), connector name( C3 ) [ 26.887469] scsi 0:0:105:0: serial_number(ZA1B731E0000R833UYGB) [ 26.893388] scsi 0:0:105:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.916663] mpt3sas_cm0: detecting: handle(0x00a7), sas_address(0x5000c500a6106479), phy(44) [ 26.925103] mpt3sas_cm0: REPORT_LUNS: handle(0x00a7), retries(0) [ 26.936408] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a7), lun(0) [ 26.971007] scsi 0:0:106:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 26.979650] scsi 0:0:106:0: SSP: handle(0x00a7), sas_addr(0x5000c500a6106479), phy(44), device_name(0x5000c500a6106478) [ 26.990421] scsi 0:0:106:0: enclosure logical id(0x500163600190cd7d), slot(44) [ 26.997725] scsi 0:0:106:0: enclosure level(0x0000), connector name( C3 ) [ 27.004616] scsi 0:0:106:0: serial_number(ZA1B72310000R8314SBW) [ 27.010540] scsi 0:0:106:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.034072] mpt3sas_cm0: detecting: handle(0x00a8), sas_address(0x5000c500a610264d), phy(45) [ 27.042514] mpt3sas_cm0: REPORT_LUNS: handle(0x00a8), retries(0) [ 27.064491] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a8), lun(0) [ 27.097402] scsi 0:0:107:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 27.106044] scsi 0:0:107:0: SSP: handle(0x00a8), sas_addr(0x5000c500a610264d), phy(45), device_name(0x5000c500a610264c) [ 27.116818] scsi 0:0:107:0: enclosure logical id(0x500163600190cd7d), slot(45) [ 27.124125] scsi 0:0:107:0: enclosure level(0x0000), connector name( C3 ) [ 27.131017] scsi 0:0:107:0: serial_number(ZA1B72XL0000R833UYDL) [ 27.136937] scsi 0:0:107:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.160079] mpt3sas_cm0: detecting: handle(0x00a9), sas_address(0x5000c500954d4401), phy(46) [ 27.168520] mpt3sas_cm0: REPORT_LUNS: handle(0x00a9), retries(0) [ 27.189535] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a9), lun(0) [ 27.222106] scsi 0:0:108:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 27.230745] scsi 0:0:108:0: SSP: handle(0x00a9), sas_addr(0x5000c500954d4401), phy(46), device_name(0x5000c500954d4400) [ 27.241519] scsi 0:0:108:0: enclosure logical id(0x500163600190cd7d), slot(46) [ 27.248823] scsi 0:0:108:0: enclosure level(0x0000), connector name( C3 ) [ 27.255715] scsi 0:0:108:0: serial_number(ZA1AHBRF0000R8318PP8) [ 27.261636] scsi 0:0:108:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.285083] mpt3sas_cm0: detecting: handle(0x00aa), sas_address(0x5000c500957eb4c5), phy(47) [ 27.293523] mpt3sas_cm0: REPORT_LUNS: handle(0x00aa), retries(0) [ 27.310514] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00aa), lun(0) [ 27.351208] scsi 0:0:109:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 27.359855] scsi 0:0:109:0: SSP: handle(0x00aa), sas_addr(0x5000c500957eb4c5), phy(47), device_name(0x5000c500957eb4c4) [ 27.370629] scsi 0:0:109:0: enclosure logical id(0x500163600190cd7d), slot(47) [ 27.377936] scsi 0:0:109:0: enclosure level(0x0000), connector name( C3 ) [ 27.384826] scsi 0:0:109:0: serial_number(ZA1B6AXY0000R8332SL8) [ 27.390746] scsi 0:0:109:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.414085] mpt3sas_cm0: detecting: handle(0x00b3), sas_address(0x5000c500954328f5), phy(48) [ 27.422524] mpt3sas_cm0: REPORT_LUNS: handle(0x00b3), retries(0) [ 27.440679] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b3), lun(0) [ 27.472531] scsi 0:0:110:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 27.481179] scsi 0:0:110:0: SSP: handle(0x00b3), sas_addr(0x5000c500954328f5), phy(48), device_name(0x5000c500954328f4) [ 27.491948] scsi 0:0:110:0: enclosure logical id(0x500163600190cd7d), slot(48) [ 27.499255] scsi 0:0:110:0: enclosure level(0x0000), connector name( C3 ) [ 27.506148] scsi 0:0:110:0: serial_number(ZA1AGMWA0000C8316H64) [ 27.512066] scsi 0:0:110:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.536095] mpt3sas_cm0: detecting: handle(0x00b4), sas_address(0x5000c50095432165), phy(49) [ 27.544536] mpt3sas_cm0: REPORT_LUNS: handle(0x00b4), retries(0) [ 27.566437] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b4), lun(0) [ 27.603109] scsi 0:0:111:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 27.611759] scsi 0:0:111:0: SSP: handle(0x00b4), sas_addr(0x5000c50095432165), phy(49), device_name(0x5000c50095432164) [ 27.622533] scsi 0:0:111:0: enclosure logical id(0x500163600190cd7d), slot(49) [ 27.629837] scsi 0:0:111:0: enclosure level(0x0000), connector name( C3 ) [ 27.636729] scsi 0:0:111:0: serial_number(ZA1AGGY80000C724RXCM) [ 27.642650] scsi 0:0:111:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.666092] mpt3sas_cm0: detecting: handle(0x00b5), sas_address(0x5000c500954313bd), phy(50) [ 27.674532] mpt3sas_cm0: REPORT_LUNS: handle(0x00b5), retries(0) [ 27.692318] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b5), lun(0) [ 27.733413] scsi 0:0:112:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 27.742060] scsi 0:0:112:0: SSP: handle(0x00b5), sas_addr(0x5000c500954313bd), phy(50), device_name(0x5000c500954313bc) [ 27.752832] scsi 0:0:112:0: enclosure logical id(0x500163600190cd7d), slot(50) [ 27.760138] scsi 0:0:112:0: enclosure level(0x0000), connector name( C3 ) [ 27.767028] scsi 0:0:112:0: serial_number(ZA1AGHD30000C8316HPY) [ 27.772947] scsi 0:0:112:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.796091] mpt3sas_cm0: detecting: handle(0x00b6), sas_address(0x5000c50095431581), phy(51) [ 27.804526] mpt3sas_cm0: REPORT_LUNS: handle(0x00b6), retries(0) [ 27.811088] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b6), lun(0) [ 28.372599] scsi 0:0:113:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.381404] scsi 0:0:113:0: SSP: handle(0x00b6), sas_addr(0x5000c50095431581), phy(51), device_name(0x5000c50095431580) [ 28.392181] scsi 0:0:113:0: enclosure logical id(0x500163600190cd7d), slot(51) [ 28.399485] scsi 0:0:113:0: enclosure level(0x0000), connector name( C3 ) [ 28.406377] scsi 0:0:113:0: serial_number(ZA1AGHB90000R7245BUM) [ 28.412298] scsi 0:0:113:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.820159] mpt3sas_cm0: detecting: handle(0x00b7), sas_address(0x5000c50095430f29), phy(52) [ 28.828614] mpt3sas_cm0: REPORT_LUNS: handle(0x00b7), retries(0) [ 28.841567] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b7), lun(0) [ 28.880577] scsi 0:0:114:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 28.889224] scsi 0:0:114:0: SSP: handle(0x00b7), sas_addr(0x5000c50095430f29), phy(52), device_name(0x5000c50095430f28) [ 28.900000] scsi 0:0:114:0: enclosure logical id(0x500163600190cd7d), slot(52) [ 28.907306] scsi 0:0:114:0: enclosure level(0x0000), connector name( C3 ) [ 28.914198] scsi 0:0:114:0: serial_number(ZA1AGHJF0000C8316HQY) [ 28.920119] scsi 0:0:114:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.943121] mpt3sas_cm0: detecting: handle(0x00b8), sas_address(0x5000c500954327f1), phy(53) [ 28.951558] mpt3sas_cm0: REPORT_LUNS: handle(0x00b8), retries(0) [ 28.964385] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b8), lun(0) [ 29.005282] scsi 0:0:115:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.013930] scsi 0:0:115:0: SSP: handle(0x00b8), sas_addr(0x5000c500954327f1), phy(53), device_name(0x5000c500954327f0) [ 29.024700] scsi 0:0:115:0: enclosure logical id(0x500163600190cd7d), slot(53) [ 29.032006] scsi 0:0:115:0: enclosure level(0x0000), connector name( C3 ) [ 29.038899] scsi 0:0:115:0: serial_number(ZA1AGGP90000R814TJY2) [ 29.044818] scsi 0:0:115:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.068124] mpt3sas_cm0: detecting: handle(0x00b9), sas_address(0x5000c500954327f9), phy(54) [ 29.076559] mpt3sas_cm0: REPORT_LUNS: handle(0x00b9), retries(0) [ 29.095704] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b9), lun(0) [ 29.102790] scsi 0:0:116:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.111446] scsi 0:0:116:0: SSP: handle(0x00b9), sas_addr(0x5000c500954327f9), phy(54), device_name(0x5000c500954327f8) [ 29.122220] scsi 0:0:116:0: enclosure logical id(0x500163600190cd7d), slot(54) [ 29.129526] scsi 0:0:116:0: enclosure level(0x0000), connector name( C3 ) [ 29.136419] scsi 0:0:116:0: serial_number(ZA1AGGP70000C8316FPD) [ 29.142338] scsi 0:0:116:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.166123] mpt3sas_cm0: detecting: handle(0x00ba), sas_address(0x5000c5009543222d), phy(55) [ 29.174563] mpt3sas_cm0: REPORT_LUNS: handle(0x00ba), retries(0) [ 29.194820] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ba), lun(0) [ 29.228104] scsi 0:0:117:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.236750] scsi 0:0:117:0: SSP: handle(0x00ba), sas_addr(0x5000c5009543222d), phy(55), device_name(0x5000c5009543222c) [ 29.247526] scsi 0:0:117:0: enclosure logical id(0x500163600190cd7d), slot(55) [ 29.254832] scsi 0:0:117:0: enclosure level(0x0000), connector name( C3 ) [ 29.261724] scsi 0:0:117:0: serial_number(ZA1AGH6Q0000C8316FPY) [ 29.267645] scsi 0:0:117:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.291130] mpt3sas_cm0: detecting: handle(0x00c3), sas_address(0x5000c50095372625), phy(56) [ 29.299567] mpt3sas_cm0: REPORT_LUNS: handle(0x00c3), retries(0) [ 29.306141] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c3), lun(0) [ 29.863638] scsi 0:0:118:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 29.872434] scsi 0:0:118:0: SSP: handle(0x00c3), sas_addr(0x5000c50095372625), phy(56), device_name(0x5000c50095372624) [ 29.883208] scsi 0:0:118:0: enclosure logical id(0x500163600190cd7d), slot(56) [ 29.890514] scsi 0:0:118:0: enclosure level(0x0000), connector name( C3 ) [ 29.897408] scsi 0:0:118:0: serial_number(ZA1ABZ4Y0000R829D6VF) [ 29.903326] scsi 0:0:118:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.307187] mpt3sas_cm0: detecting: handle(0x00c4), sas_address(0x5000c50095430cb1), phy(57) [ 30.315639] mpt3sas_cm0: REPORT_LUNS: handle(0x00c4), retries(0) [ 30.334819] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c4), lun(0) [ 30.367396] scsi 0:0:119:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.376039] scsi 0:0:119:0: SSP: handle(0x00c4), sas_addr(0x5000c50095430cb1), phy(57), device_name(0x5000c50095430cb0) [ 30.386807] scsi 0:0:119:0: enclosure logical id(0x500163600190cd7d), slot(57) [ 30.394114] scsi 0:0:119:0: enclosure level(0x0000), connector name( C3 ) [ 30.401007] scsi 0:0:119:0: serial_number(ZA1AGHLV0000R823QBK4) [ 30.406926] scsi 0:0:119:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.430151] mpt3sas_cm0: detecting: handle(0x00c5), sas_address(0x5000c50095430c31), phy(58) [ 30.438589] mpt3sas_cm0: REPORT_LUNS: handle(0x00c5), retries(0) [ 30.445105] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c5), lun(0) [ 30.477066] scsi 0:0:120:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.485710] scsi 0:0:120:0: SSP: handle(0x00c5), sas_addr(0x5000c50095430c31), phy(58), device_name(0x5000c50095430c30) [ 30.496480] scsi 0:0:120:0: enclosure logical id(0x500163600190cd7d), slot(58) [ 30.503786] scsi 0:0:120:0: enclosure level(0x0000), connector name( C3 ) [ 30.510664] scsi 0:0:120:0: serial_number(ZA1AGHMK0000R824E79X) [ 30.516590] scsi 0:0:120:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.540163] mpt3sas_cm0: detecting: handle(0x00c6), sas_address(0x5000c500954327cd), phy(59) [ 30.548603] mpt3sas_cm0: REPORT_LUNS: handle(0x00c6), retries(0) [ 30.570569] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c6), lun(0) [ 30.577674] scsi 0:0:121:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.586328] scsi 0:0:121:0: SSP: handle(0x00c6), sas_addr(0x5000c500954327cd), phy(59), device_name(0x5000c500954327cc) [ 30.597102] scsi 0:0:121:0: enclosure logical id(0x500163600190cd7d), slot(59) [ 30.604408] scsi 0:0:121:0: enclosure level(0x0000), connector name( C3 ) [ 30.611301] scsi 0:0:121:0: serial_number(ZA1AGGPH0000R829CU4W) [ 30.617220] scsi 0:0:121:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.641283] mpt3sas_cm0: detecting: handle(0x00cd), sas_address(0x500163600190cd7d), phy(76) [ 30.649740] mpt3sas_cm0: REPORT_LUNS: handle(0x00cd), retries(0) [ 30.655933] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00cd), lun(0) [ 30.662410] scsi 0:0:122:0: Enclosure QCT JB4602 SIM 0 1500 PQ: 0 ANSI: 6 [ 30.670876] scsi 0:0:122:0: set ignore_delay_remove for handle(0x00cd) [ 30.677402] scsi 0:0:122:0: SES: handle(0x00cd), sas_addr(0x500163600190cd7d), phy(76), device_name(0x500163600190cd7d) [ 30.688173] scsi 0:0:122:0: enclosure logical id(0x500163600190cd7d), slot(60) [ 30.695480] scsi 0:0:122:0: enclosure level(0x0000), connector name( C3 ) [ 30.702372] scsi 0:0:122:0: serial_number(EO0152000043 ) [ 30.707944] scsi 0:0:122:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.728576] mpt3sas_cm0: expander_add: handle(0x0052), parent(0x000a), sas_addr(0x5001636001a0673f), phys(79) [ 30.749712] mpt3sas_cm0: detecting: handle(0x005b), sas_address(0x5000c500954327d2), phy(0) [ 30.758059] mpt3sas_cm0: REPORT_LUNS: handle(0x005b), retries(0) [ 30.777683] mpt3sas_cm0: TEST_UNIT_READY: handle(0x005b), lun(0) [ 30.784792] scsi 0:0:123:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.793456] scsi 0:0:123:0: SSP: handle(0x005b), sas_addr(0x5000c500954327d2), phy(0), device_name(0x5000c500954327d0) [ 30.804147] scsi 0:0:123:0: enclosure logical id(0x5001636001a0673d), slot(0) [ 30.811366] scsi 0:0:123:0: enclosure level(0x0000), connector name( C0 ) [ 30.818259] scsi 0:0:123:0: serial_number(ZA1AGGPM0000R829CTE6) [ 30.824178] scsi 0:0:123:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.847177] mpt3sas_cm0: detecting: handle(0x005c), sas_address(0x5000c5009543210e), phy(1) [ 30.855527] mpt3sas_cm0: REPORT_LUNS: handle(0x005c), retries(0) [ 30.869676] mpt3sas_cm0: TEST_UNIT_READY: handle(0x005c), lun(0) [ 30.876777] scsi 0:0:124:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.885429] scsi 0:0:124:0: SSP: handle(0x005c), sas_addr(0x5000c5009543210e), phy(1), device_name(0x5000c5009543210c) [ 30.896118] scsi 0:0:124:0: enclosure logical id(0x5001636001a0673d), slot(1) [ 30.903339] scsi 0:0:124:0: enclosure level(0x0000), connector name( C0 ) [ 30.910232] scsi 0:0:124:0: serial_number(ZA1AGGYR0000C72542A6) [ 30.916150] scsi 0:0:124:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 30.939176] mpt3sas_cm0: detecting: handle(0x005d), sas_address(0x5000c500954322be), phy(2) [ 30.947527] mpt3sas_cm0: REPORT_LUNS: handle(0x005d), retries(0) [ 30.960694] mpt3sas_cm0: TEST_UNIT_READY: handle(0x005d), lun(0) [ 30.967792] scsi 0:0:125:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 30.976450] scsi 0:0:125:0: SSP: handle(0x005d), sas_addr(0x5000c500954322be), phy(2), device_name(0x5000c500954322bc) [ 30.987138] scsi 0:0:125:0: enclosure logical id(0x5001636001a0673d), slot(2) [ 30.994359] scsi 0:0:125:0: enclosure level(0x0000), connector name( C0 ) [ 31.001252] scsi 0:0:125:0: serial_number(ZA1AGGX60000R70426B4) [ 31.007170] scsi 0:0:125:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.030171] mpt3sas_cm0: detecting: handle(0x005e), sas_address(0x5000c50095431eee), phy(3) [ 31.038524] mpt3sas_cm0: REPORT_LUNS: handle(0x005e), retries(0) [ 31.058010] mpt3sas_cm0: TEST_UNIT_READY: handle(0x005e), lun(0) [ 31.065098] scsi 0:0:126:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.073756] scsi 0:0:126:0: SSP: handle(0x005e), sas_addr(0x5000c50095431eee), phy(3), device_name(0x5000c50095431eec) [ 31.084442] scsi 0:0:126:0: enclosure logical id(0x5001636001a0673d), slot(3) [ 31.091662] scsi 0:0:126:0: enclosure level(0x0000), connector name( C0 ) [ 31.098555] scsi 0:0:126:0: serial_number(ZA1AGH0M0000C8316G8B) [ 31.104473] scsi 0:0:126:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.127170] mpt3sas_cm0: detecting: handle(0x005f), sas_address(0x5000c500954307ea), phy(4) [ 31.135520] mpt3sas_cm0: REPORT_LUNS: handle(0x005f), retries(0) [ 31.142066] mpt3sas_cm0: TEST_UNIT_READY: handle(0x005f), lun(0) [ 31.149156] scsi 0:0:127:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.157818] scsi 0:0:127:0: SSP: handle(0x005f), sas_addr(0x5000c500954307ea), phy(4), device_name(0x5000c500954307e8) [ 31.168502] scsi 0:0:127:0: enclosure logical id(0x5001636001a0673d), slot(4) [ 31.175722] scsi 0:0:127:0: enclosure level(0x0000), connector name( C0 ) [ 31.182614] scsi 0:0:127:0: serial_number(ZA1AG8FH0000R8301BKX) [ 31.188534] scsi 0:0:127:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.211175] mpt3sas_cm0: detecting: handle(0x0060), sas_address(0x5000c500954d3c9e), phy(5) [ 31.219529] mpt3sas_cm0: REPORT_LUNS: handle(0x0060), retries(0) [ 31.237825] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0060), lun(0) [ 31.280826] scsi 0:0:128:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.289473] scsi 0:0:128:0: SSP: handle(0x0060), sas_addr(0x5000c500954d3c9e), phy(5), device_name(0x5000c500954d3c9c) [ 31.300162] scsi 0:0:128:0: enclosure logical id(0x5001636001a0673d), slot(5) [ 31.307381] scsi 0:0:128:0: enclosure level(0x0000), connector name( C0 ) [ 31.314274] scsi 0:0:128:0: serial_number(ZA1ALXTG0000R8314UFV) [ 31.320193] scsi 0:0:128:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.343176] mpt3sas_cm0: detecting: handle(0x0061), sas_address(0x5000c5009537de22), phy(6) [ 31.351526] mpt3sas_cm0: REPORT_LUNS: handle(0x0061), retries(0) [ 31.372896] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0061), lun(0) [ 31.379991] scsi 0:0:129:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.388649] scsi 0:0:129:0: SSP: handle(0x0061), sas_addr(0x5000c5009537de22), phy(6), device_name(0x5000c5009537de20) [ 31.399335] scsi 0:0:129:0: enclosure logical id(0x5001636001a0673d), slot(6) [ 31.406556] scsi 0:0:129:0: enclosure level(0x0000), connector name( C0 ) [ 31.413449] scsi 0:0:129:0: serial_number(ZA1ACKPT0000R828A04K) [ 31.419370] scsi 0:0:129:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.443182] mpt3sas_cm0: detecting: handle(0x0062), sas_address(0x5000c500954cd562), phy(7) [ 31.451532] mpt3sas_cm0: REPORT_LUNS: handle(0x0062), retries(0) [ 31.470131] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0062), lun(0) [ 31.477220] scsi 0:0:130:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.485876] scsi 0:0:130:0: SSP: handle(0x0062), sas_addr(0x5000c500954cd562), phy(7), device_name(0x5000c500954cd560) [ 31.496562] scsi 0:0:130:0: enclosure logical id(0x5001636001a0673d), slot(7) [ 31.503780] scsi 0:0:130:0: enclosure level(0x0000), connector name( C0 ) [ 31.510672] scsi 0:0:130:0: serial_number(ZA1ALGB20000R8314QNY) [ 31.516596] scsi 0:0:130:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.541187] mpt3sas_cm0: detecting: handle(0x006b), sas_address(0x5000c500954d2eda), phy(8) [ 31.549542] mpt3sas_cm0: REPORT_LUNS: handle(0x006b), retries(0) [ 31.556095] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006b), lun(0) [ 31.563191] scsi 0:0:131:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.571851] scsi 0:0:131:0: SSP: handle(0x006b), sas_addr(0x5000c500954d2eda), phy(8), device_name(0x5000c500954d2ed8) [ 31.582539] scsi 0:0:131:0: enclosure logical id(0x5001636001a0673d), slot(8) [ 31.589757] scsi 0:0:131:0: enclosure level(0x0000), connector name( C0 ) [ 31.596648] scsi 0:0:131:0: serial_number(ZA1ALV4P0000R830XC2X) [ 31.602569] scsi 0:0:131:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.625182] mpt3sas_cm0: detecting: handle(0x006c), sas_address(0x5000c50095430afe), phy(9) [ 31.633532] mpt3sas_cm0: REPORT_LUNS: handle(0x006c), retries(0) [ 31.647202] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006c), lun(0) [ 31.654289] scsi 0:0:132:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.662950] scsi 0:0:132:0: SSP: handle(0x006c), sas_addr(0x5000c50095430afe), phy(9), device_name(0x5000c50095430afc) [ 31.673635] scsi 0:0:132:0: enclosure logical id(0x5001636001a0673d), slot(9) [ 31.680854] scsi 0:0:132:0: enclosure level(0x0000), connector name( C0 ) [ 31.687749] scsi 0:0:132:0: serial_number(ZA1ABZKP0000R8301BVX) [ 31.693668] scsi 0:0:132:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.717186] mpt3sas_cm0: detecting: handle(0x006d), sas_address(0x5000c500954303fa), phy(10) [ 31.725624] mpt3sas_cm0: REPORT_LUNS: handle(0x006d), retries(0) [ 31.739432] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006d), lun(0) [ 31.746528] scsi 0:0:133:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.755184] scsi 0:0:133:0: SSP: handle(0x006d), sas_addr(0x5000c500954303fa), phy(10), device_name(0x5000c500954303f8) [ 31.765955] scsi 0:0:133:0: enclosure logical id(0x5001636001a0673d), slot(10) [ 31.773262] scsi 0:0:133:0: enclosure level(0x0000), connector name( C0 ) [ 31.780154] scsi 0:0:133:0: serial_number(ZA1AG8K80000C8316HBT) [ 31.786074] scsi 0:0:133:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.809191] mpt3sas_cm0: detecting: handle(0x006e), sas_address(0x5000c50095431ea6), phy(11) [ 31.817625] mpt3sas_cm0: REPORT_LUNS: handle(0x006e), retries(0) [ 31.828799] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006e), lun(0) [ 31.835881] scsi 0:0:134:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.844542] scsi 0:0:134:0: SSP: handle(0x006e), sas_addr(0x5000c50095431ea6), phy(11), device_name(0x5000c50095431ea4) [ 31.855311] scsi 0:0:134:0: enclosure logical id(0x5001636001a0673d), slot(11) [ 31.862617] scsi 0:0:134:0: enclosure level(0x0000), connector name( C0 ) [ 31.869510] scsi 0:0:134:0: serial_number(ZA1AGN3P0000R829D5A4) [ 31.875431] scsi 0:0:134:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.898189] mpt3sas_cm0: detecting: handle(0x006f), sas_address(0x5000c5009542c79e), phy(12) [ 31.906626] mpt3sas_cm0: REPORT_LUNS: handle(0x006f), retries(0) [ 31.918630] mpt3sas_cm0: TEST_UNIT_READY: handle(0x006f), lun(0) [ 31.925729] scsi 0:0:135:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 31.934388] scsi 0:0:135:0: SSP: handle(0x006f), sas_addr(0x5000c5009542c79e), phy(12), device_name(0x5000c5009542c79c) [ 31.945162] scsi 0:0:135:0: enclosure logical id(0x5001636001a0673d), slot(12) [ 31.952468] scsi 0:0:135:0: enclosure level(0x0000), connector name( C0 ) [ 31.959361] scsi 0:0:135:0: serial_number(ZA1AGACY0000R8301U8F) [ 31.965280] scsi 0:0:135:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 31.988191] mpt3sas_cm0: detecting: handle(0x0070), sas_address(0x5000c50095431552), phy(13) [ 31.996630] mpt3sas_cm0: REPORT_LUNS: handle(0x0070), retries(0) [ 32.003180] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0070), lun(0) [ 32.010263] scsi 0:0:136:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.018923] scsi 0:0:136:0: SSP: handle(0x0070), sas_addr(0x5000c50095431552), phy(13), device_name(0x5000c50095431550) [ 32.029698] scsi 0:0:136:0: enclosure logical id(0x5001636001a0673d), slot(13) [ 32.037004] scsi 0:0:136:0: enclosure level(0x0000), connector name( C0 ) [ 32.043895] scsi 0:0:136:0: serial_number(ZA1AFWDA0000C83153E9) [ 32.049815] scsi 0:0:136:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.073195] mpt3sas_cm0: detecting: handle(0x0071), sas_address(0x5000c5009543270e), phy(14) [ 32.081637] mpt3sas_cm0: REPORT_LUNS: handle(0x0071), retries(0) [ 32.098148] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0071), lun(0) [ 32.105246] scsi 0:0:137:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.113908] scsi 0:0:137:0: SSP: handle(0x0071), sas_addr(0x5000c5009543270e), phy(14), device_name(0x5000c5009543270c) [ 32.124680] scsi 0:0:137:0: enclosure logical id(0x5001636001a0673d), slot(14) [ 32.131984] scsi 0:0:137:0: enclosure level(0x0000), connector name( C0 ) [ 32.138875] scsi 0:0:137:0: serial_number(ZA1AGGQF0000C83153PC) [ 32.144796] scsi 0:0:137:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.168199] mpt3sas_cm0: detecting: handle(0x0072), sas_address(0x5000c5009543269e), phy(15) [ 32.176634] mpt3sas_cm0: REPORT_LUNS: handle(0x0072), retries(0) [ 32.192239] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0072), lun(0) [ 32.199331] scsi 0:0:138:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.207992] scsi 0:0:138:0: SSP: handle(0x0072), sas_addr(0x5000c5009543269e), phy(15), device_name(0x5000c5009543269c) [ 32.218767] scsi 0:0:138:0: enclosure logical id(0x5001636001a0673d), slot(15) [ 32.226072] scsi 0:0:138:0: enclosure level(0x0000), connector name( C0 ) [ 32.232966] scsi 0:0:138:0: serial_number(ZA1AGGVJ0000R829CUL1) [ 32.238886] scsi 0:0:138:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.262202] mpt3sas_cm0: detecting: handle(0x007b), sas_address(0x5000c500954306fa), phy(16) [ 32.270644] mpt3sas_cm0: REPORT_LUNS: handle(0x007b), retries(0) [ 32.281863] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007b), lun(0) [ 32.288965] scsi 0:0:139:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.297631] scsi 0:0:139:0: SSP: handle(0x007b), sas_addr(0x5000c500954306fa), phy(16), device_name(0x5000c500954306f8) [ 32.308399] scsi 0:0:139:0: enclosure logical id(0x5001636001a0673d), slot(16) [ 32.315706] scsi 0:0:139:0: enclosure level(0x0000), connector name( C0 ) [ 32.322598] scsi 0:0:139:0: serial_number(ZA1AG8GS0000R8301BGG) [ 32.328519] scsi 0:0:139:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.351199] mpt3sas_cm0: detecting: handle(0x007c), sas_address(0x5000c500954d3a86), phy(17) [ 32.359636] mpt3sas_cm0: REPORT_LUNS: handle(0x007c), retries(0) [ 32.376008] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007c), lun(0) [ 32.423169] scsi 0:0:140:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.431821] scsi 0:0:140:0: SSP: handle(0x007c), sas_addr(0x5000c500954d3a86), phy(17), device_name(0x5000c500954d3a84) [ 32.442589] scsi 0:0:140:0: enclosure logical id(0x5001636001a0673d), slot(17) [ 32.449897] scsi 0:0:140:0: enclosure level(0x0000), connector name( C0 ) [ 32.456788] scsi 0:0:140:0: serial_number(ZA1ALTNS0000R8314U9A) [ 32.462709] scsi 0:0:140:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.486203] mpt3sas_cm0: detecting: handle(0x007d), sas_address(0x5000c500954d972a), phy(18) [ 32.494639] mpt3sas_cm0: REPORT_LUNS: handle(0x007d), retries(0) [ 32.509461] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007d), lun(0) [ 32.516563] scsi 0:0:141:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.525222] scsi 0:0:141:0: SSP: handle(0x007d), sas_addr(0x5000c500954d972a), phy(18), device_name(0x5000c500954d9728) [ 32.535993] scsi 0:0:141:0: enclosure logical id(0x5001636001a0673d), slot(18) [ 32.543300] scsi 0:0:141:0: enclosure level(0x0000), connector name( C0 ) [ 32.550192] scsi 0:0:141:0: serial_number(ZA1AM3T00000R8314URZ) [ 32.556115] scsi 0:0:141:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.579212] mpt3sas_cm0: detecting: handle(0x007e), sas_address(0x5000c5009543190e), phy(19) [ 32.587655] mpt3sas_cm0: REPORT_LUNS: handle(0x007e), retries(0) [ 32.605168] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007e), lun(0) [ 32.612438] scsi 0:0:142:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.621097] scsi 0:0:142:0: SSP: handle(0x007e), sas_addr(0x5000c5009543190e), phy(19), device_name(0x5000c5009543190c) [ 32.631867] scsi 0:0:142:0: enclosure logical id(0x5001636001a0673d), slot(19) [ 32.639173] scsi 0:0:142:0: enclosure level(0x0000), connector name( C0 ) [ 32.646062] scsi 0:0:142:0: serial_number(ZA1AFH3W0000C8316GJ6) [ 32.651985] scsi 0:0:142:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.675210] mpt3sas_cm0: detecting: handle(0x007f), sas_address(0x5000c500954325c2), phy(20) [ 32.683648] mpt3sas_cm0: REPORT_LUNS: handle(0x007f), retries(0) [ 32.700232] mpt3sas_cm0: TEST_UNIT_READY: handle(0x007f), lun(0) [ 32.707330] scsi 0:0:143:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.715990] scsi 0:0:143:0: SSP: handle(0x007f), sas_addr(0x5000c500954325c2), phy(20), device_name(0x5000c500954325c0) [ 32.726760] scsi 0:0:143:0: enclosure logical id(0x5001636001a0673d), slot(20) [ 32.734066] scsi 0:0:143:0: enclosure level(0x0000), connector name( C0 ) [ 32.740958] scsi 0:0:143:0: serial_number(ZA1AGGV20000R829CG7B) [ 32.746878] scsi 0:0:143:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.770217] mpt3sas_cm0: detecting: handle(0x0080), sas_address(0x5000c50095431ef6), phy(21) [ 32.778656] mpt3sas_cm0: REPORT_LUNS: handle(0x0080), retries(0) [ 32.796064] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0080), lun(0) [ 32.803169] scsi 0:0:144:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.811832] scsi 0:0:144:0: SSP: handle(0x0080), sas_addr(0x5000c50095431ef6), phy(21), device_name(0x5000c50095431ef4) [ 32.822609] scsi 0:0:144:0: enclosure logical id(0x5001636001a0673d), slot(21) [ 32.829913] scsi 0:0:144:0: enclosure level(0x0000), connector name( C0 ) [ 32.836804] scsi 0:0:144:0: serial_number(ZA1AGH4R0000C72541L0) [ 32.842724] scsi 0:0:144:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.866808] mpt3sas_cm0: detecting: handle(0x0081), sas_address(0x5000c500954328c2), phy(22) [ 32.875248] mpt3sas_cm0: REPORT_LUNS: handle(0x0081), retries(0) [ 32.890135] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0081), lun(0) [ 32.897230] scsi 0:0:145:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 32.905898] scsi 0:0:145:0: SSP: handle(0x0081), sas_addr(0x5000c500954328c2), phy(22), device_name(0x5000c500954328c0) [ 32.916670] scsi 0:0:145:0: enclosure logical id(0x5001636001a0673d), slot(22) [ 32.923975] scsi 0:0:145:0: enclosure level(0x0000), connector name( C0 ) [ 32.930865] scsi 0:0:145:0: serial_number(ZA1AGGMS0000C8316FAT) [ 32.936788] scsi 0:0:145:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 32.960218] mpt3sas_cm0: detecting: handle(0x0082), sas_address(0x5000c50095431c8e), phy(23) [ 32.968659] mpt3sas_cm0: REPORT_LUNS: handle(0x0082), retries(0) [ 32.985277] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0082), lun(0) [ 32.992383] scsi 0:0:146:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.001039] scsi 0:0:146:0: SSP: handle(0x0082), sas_addr(0x5000c50095431c8e), phy(23), device_name(0x5000c50095431c8c) [ 33.011814] scsi 0:0:146:0: enclosure logical id(0x5001636001a0673d), slot(23) [ 33.019121] scsi 0:0:146:0: enclosure level(0x0000), connector name( C0 ) [ 33.026012] scsi 0:0:146:0: serial_number(ZA1AGH310000R727Y825) [ 33.031933] scsi 0:0:146:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.055218] mpt3sas_cm0: detecting: handle(0x008b), sas_address(0x5000c50095430e76), phy(24) [ 33.063658] mpt3sas_cm0: REPORT_LUNS: handle(0x008b), retries(0) [ 33.082764] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008b), lun(0) [ 33.089859] scsi 0:0:147:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.098516] scsi 0:0:147:0: SSP: handle(0x008b), sas_addr(0x5000c50095430e76), phy(24), device_name(0x5000c50095430e74) [ 33.109293] scsi 0:0:147:0: enclosure logical id(0x5001636001a0673d), slot(24) [ 33.116597] scsi 0:0:147:0: enclosure level(0x0000), connector name( C0 ) [ 33.123489] scsi 0:0:147:0: serial_number(ZA1AGHKG0000R829D7P9) [ 33.129410] scsi 0:0:147:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.152217] mpt3sas_cm0: detecting: handle(0x008c), sas_address(0x5000c500954cec32), phy(25) [ 33.160659] mpt3sas_cm0: REPORT_LUNS: handle(0x008c), retries(0) [ 33.175053] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008c), lun(0) [ 33.182147] scsi 0:0:148:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.190808] scsi 0:0:148:0: SSP: handle(0x008c), sas_addr(0x5000c500954cec32), phy(25), device_name(0x5000c500954cec30) [ 33.201577] scsi 0:0:148:0: enclosure logical id(0x5001636001a0673d), slot(25) [ 33.208882] scsi 0:0:148:0: enclosure level(0x0000), connector name( C0 ) [ 33.215777] scsi 0:0:148:0: serial_number(ZA1ALFQC0000R8314S0C) [ 33.221696] scsi 0:0:148:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.245227] mpt3sas_cm0: detecting: handle(0x008d), sas_address(0x5000c500954d2d0e), phy(26) [ 33.253662] mpt3sas_cm0: REPORT_LUNS: handle(0x008d), retries(0) [ 33.272086] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008d), lun(0) [ 33.279213] scsi 0:0:149:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.287876] scsi 0:0:149:0: SSP: handle(0x008d), sas_addr(0x5000c500954d2d0e), phy(26), device_name(0x5000c500954d2d0c) [ 33.298648] scsi 0:0:149:0: enclosure logical id(0x5001636001a0673d), slot(26) [ 33.305952] scsi 0:0:149:0: enclosure level(0x0000), connector name( C0 ) [ 33.312844] scsi 0:0:149:0: serial_number(ZA1ALV770000R8314TF5) [ 33.318765] scsi 0:0:149:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.342230] mpt3sas_cm0: detecting: handle(0x008e), sas_address(0x5000c500954de6be), phy(27) [ 33.350671] mpt3sas_cm0: REPORT_LUNS: handle(0x008e), retries(0) [ 33.357145] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008e), lun(0) [ 33.370643] scsi 0:0:150:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.379282] scsi 0:0:150:0: SSP: handle(0x008e), sas_addr(0x5000c500954de6be), phy(27), device_name(0x5000c500954de6bc) [ 33.390056] scsi 0:0:150:0: enclosure logical id(0x5001636001a0673d), slot(27) [ 33.397362] scsi 0:0:150:0: enclosure level(0x0000), connector name( C0 ) [ 33.404254] scsi 0:0:150:0: serial_number(ZA1AM5MM0000R8317PAM) [ 33.410175] scsi 0:0:150:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.436236] mpt3sas_cm0: detecting: handle(0x008f), sas_address(0x5000c500954321ce), phy(28) [ 33.444674] mpt3sas_cm0: REPORT_LUNS: handle(0x008f), retries(0) [ 33.460072] mpt3sas_cm0: TEST_UNIT_READY: handle(0x008f), lun(0) [ 33.467163] scsi 0:0:151:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.475824] scsi 0:0:151:0: SSP: handle(0x008f), sas_addr(0x5000c500954321ce), phy(28), device_name(0x5000c500954321cc) [ 33.486595] scsi 0:0:151:0: enclosure logical id(0x5001636001a0673d), slot(28) [ 33.493902] scsi 0:0:151:0: enclosure level(0x0000), connector name( C0 ) [ 33.500794] scsi 0:0:151:0: serial_number(ZA1AGN0J0000R8288A2R) [ 33.506715] scsi 0:0:151:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.533219] mpt3sas_cm0: detecting: handle(0x0090), sas_address(0x5000c5009542a60e), phy(29) [ 33.541668] mpt3sas_cm0: REPORT_LUNS: handle(0x0090), retries(0) [ 33.563376] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0090), lun(0) [ 33.570484] scsi 0:0:152:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.579144] scsi 0:0:152:0: SSP: handle(0x0090), sas_addr(0x5000c5009542a60e), phy(29), device_name(0x5000c5009542a60c) [ 33.589915] scsi 0:0:152:0: enclosure logical id(0x5001636001a0673d), slot(29) [ 33.597222] scsi 0:0:152:0: enclosure level(0x0000), connector name( C0 ) [ 33.604115] scsi 0:0:152:0: serial_number(ZA1AG6DF0000R829CG6J) [ 33.610033] scsi 0:0:152:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.633226] mpt3sas_cm0: detecting: handle(0x0091), sas_address(0x5000c5009543226a), phy(30) [ 33.641661] mpt3sas_cm0: REPORT_LUNS: handle(0x0091), retries(0) [ 33.653125] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0091), lun(0) [ 33.660224] scsi 0:0:153:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.668881] scsi 0:0:153:0: SSP: handle(0x0091), sas_addr(0x5000c5009543226a), phy(30), device_name(0x5000c50095432268) [ 33.679652] scsi 0:0:153:0: enclosure logical id(0x5001636001a0673d), slot(30) [ 33.686959] scsi 0:0:153:0: enclosure level(0x0000), connector name( C0 ) [ 33.693848] scsi 0:0:153:0: serial_number(ZA1AGGXE0000R735TYMG) [ 33.699771] scsi 0:0:153:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.723230] mpt3sas_cm0: detecting: handle(0x0092), sas_address(0x5000c500953816ee), phy(31) [ 33.731668] mpt3sas_cm0: REPORT_LUNS: handle(0x0092), retries(0) [ 33.749257] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0092), lun(0) [ 33.756357] scsi 0:0:154:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.765016] scsi 0:0:154:0: SSP: handle(0x0092), sas_addr(0x5000c500953816ee), phy(31), device_name(0x5000c500953816ec) [ 33.775786] scsi 0:0:154:0: enclosure logical id(0x5001636001a0673d), slot(31) [ 33.783092] scsi 0:0:154:0: enclosure level(0x0000), connector name( C0 ) [ 33.789984] scsi 0:0:154:0: serial_number(ZA1ACHTF0000R82892CY) [ 33.795905] scsi 0:0:154:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.819235] mpt3sas_cm0: detecting: handle(0x009b), sas_address(0x5000c500954321a6), phy(32) [ 33.827670] mpt3sas_cm0: REPORT_LUNS: handle(0x009b), retries(0) [ 33.845900] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009b), lun(0) [ 33.852993] scsi 0:0:155:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.861649] scsi 0:0:155:0: SSP: handle(0x009b), sas_addr(0x5000c500954321a6), phy(32), device_name(0x5000c500954321a4) [ 33.872420] scsi 0:0:155:0: enclosure logical id(0x5001636001a0673d), slot(32) [ 33.879727] scsi 0:0:155:0: enclosure level(0x0000), connector name( C0 ) [ 33.886620] scsi 0:0:155:0: serial_number(ZA1AGGY00000C724RWTP) [ 33.892540] scsi 0:0:155:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 33.918232] mpt3sas_cm0: detecting: handle(0x009c), sas_address(0x5000c5009538bc36), phy(33) [ 33.926687] mpt3sas_cm0: REPORT_LUNS: handle(0x009c), retries(0) [ 33.933215] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009c), lun(0) [ 33.940295] scsi 0:0:156:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 33.948948] scsi 0:0:156:0: SSP: handle(0x009c), sas_addr(0x5000c5009538bc36), phy(33), device_name(0x5000c5009538bc34) [ 33.959725] scsi 0:0:156:0: enclosure logical id(0x5001636001a0673d), slot(33) [ 33.967029] scsi 0:0:156:0: enclosure level(0x0000), connector name( C0 ) [ 33.973920] scsi 0:0:156:0: serial_number(ZA1A9XY50000R8289594) [ 33.979842] scsi 0:0:156:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.003240] mpt3sas_cm0: detecting: handle(0x009d), sas_address(0x5000c500954325d6), phy(34) [ 34.011678] mpt3sas_cm0: REPORT_LUNS: handle(0x009d), retries(0) [ 34.023171] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009d), lun(0) [ 34.030264] scsi 0:0:157:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.038922] scsi 0:0:157:0: SSP: handle(0x009d), sas_addr(0x5000c500954325d6), phy(34), device_name(0x5000c500954325d4) [ 34.049694] scsi 0:0:157:0: enclosure logical id(0x5001636001a0673d), slot(34) [ 34.057001] scsi 0:0:157:0: enclosure level(0x0000), connector name( C0 ) [ 34.063892] scsi 0:0:157:0: serial_number(ZA1AGGSE0000R829CGBW) [ 34.069812] scsi 0:0:157:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.096245] mpt3sas_cm0: detecting: handle(0x009e), sas_address(0x5000c50095430222), phy(35) [ 34.104685] mpt3sas_cm0: REPORT_LUNS: handle(0x009e), retries(0) [ 34.125232] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009e), lun(0) [ 34.132340] scsi 0:0:158:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.141008] scsi 0:0:158:0: SSP: handle(0x009e), sas_addr(0x5000c50095430222), phy(35), device_name(0x5000c50095430220) [ 34.151783] scsi 0:0:158:0: enclosure logical id(0x5001636001a0673d), slot(35) [ 34.159089] scsi 0:0:158:0: enclosure level(0x0000), connector name( C0 ) [ 34.165979] scsi 0:0:158:0: serial_number(ZA1AG8NE0000C8316HG9) [ 34.171900] scsi 0:0:158:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.195251] mpt3sas_cm0: detecting: handle(0x009f), sas_address(0x5000c500953ad692), phy(36) [ 34.203694] mpt3sas_cm0: REPORT_LUNS: handle(0x009f), retries(0) [ 34.218395] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009f), lun(0) [ 34.225491] scsi 0:0:159:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.234152] scsi 0:0:159:0: SSP: handle(0x009f), sas_addr(0x5000c500953ad692), phy(36), device_name(0x5000c500953ad690) [ 34.244926] scsi 0:0:159:0: enclosure logical id(0x5001636001a0673d), slot(36) [ 34.252231] scsi 0:0:159:0: enclosure level(0x0000), connector name( C0 ) [ 34.259123] scsi 0:0:159:0: serial_number(ZA1ADPEX0000R8301P5K) [ 34.265043] scsi 0:0:159:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.288249] mpt3sas_cm0: detecting: handle(0x00a0), sas_address(0x5000c500953ad5b6), phy(37) [ 34.296689] mpt3sas_cm0: REPORT_LUNS: handle(0x00a0), retries(0) [ 34.312479] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a0), lun(0) [ 34.319585] scsi 0:0:160:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.328241] scsi 0:0:160:0: SSP: handle(0x00a0), sas_addr(0x5000c500953ad5b6), phy(37), device_name(0x5000c500953ad5b4) [ 34.339012] scsi 0:0:160:0: enclosure logical id(0x5001636001a0673d), slot(37) [ 34.346319] scsi 0:0:160:0: enclosure level(0x0000), connector name( C0 ) [ 34.353212] scsi 0:0:160:0: serial_number(ZA1ADPDH0000R83027NK) [ 34.359130] scsi 0:0:160:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.385256] mpt3sas_cm0: detecting: handle(0x00a1), sas_address(0x5000c50095431bf2), phy(38) [ 34.393698] mpt3sas_cm0: REPORT_LUNS: handle(0x00a1), retries(0) [ 34.411089] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a1), lun(0) [ 34.418177] scsi 0:0:161:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.426842] scsi 0:0:161:0: SSP: handle(0x00a1), sas_addr(0x5000c50095431bf2), phy(38), device_name(0x5000c50095431bf0) [ 34.437618] scsi 0:0:161:0: enclosure logical id(0x5001636001a0673d), slot(38) [ 34.444922] scsi 0:0:161:0: enclosure level(0x0000), connector name( C0 ) [ 34.451813] scsi 0:0:161:0: serial_number(ZA1AGH5T0000R8300NS1) [ 34.457735] scsi 0:0:161:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.510252] mpt3sas_cm0: detecting: handle(0x00a2), sas_address(0x5000c50095430ede), phy(39) [ 34.518694] mpt3sas_cm0: REPORT_LUNS: handle(0x00a2), retries(0) [ 34.537946] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a2), lun(0) [ 34.545063] scsi 0:0:162:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.553725] scsi 0:0:162:0: SSP: handle(0x00a2), sas_addr(0x5000c50095430ede), phy(39), device_name(0x5000c50095430edc) [ 34.564500] scsi 0:0:162:0: enclosure logical id(0x5001636001a0673d), slot(39) [ 34.571806] scsi 0:0:162:0: enclosure level(0x0000), connector name( C0 ) [ 34.578698] scsi 0:0:162:0: serial_number(ZA1AGHK20000R829D817) [ 34.584619] scsi 0:0:162:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.607256] mpt3sas_cm0: detecting: handle(0x00ab), sas_address(0x5000c5009543076a), phy(40) [ 34.615694] mpt3sas_cm0: REPORT_LUNS: handle(0x00ab), retries(0) [ 34.630179] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ab), lun(0) [ 34.637290] scsi 0:0:163:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.645952] scsi 0:0:163:0: SSP: handle(0x00ab), sas_addr(0x5000c5009543076a), phy(40), device_name(0x5000c50095430768) [ 34.656725] scsi 0:0:163:0: enclosure logical id(0x5001636001a0673d), slot(40) [ 34.664031] scsi 0:0:163:0: enclosure level(0x0000), connector name( C0 ) [ 34.670922] scsi 0:0:163:0: serial_number(ZA1AG3EV0000R82513S7) [ 34.676843] scsi 0:0:163:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.700259] mpt3sas_cm0: detecting: handle(0x00ac), sas_address(0x5000c50095431f32), phy(41) [ 34.708698] mpt3sas_cm0: REPORT_LUNS: handle(0x00ac), retries(0) [ 34.727110] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ac), lun(0) [ 34.734211] scsi 0:0:164:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.742869] scsi 0:0:164:0: SSP: handle(0x00ac), sas_addr(0x5000c50095431f32), phy(41), device_name(0x5000c50095431f30) [ 34.753638] scsi 0:0:164:0: enclosure logical id(0x5001636001a0673d), slot(41) [ 34.760944] scsi 0:0:164:0: enclosure level(0x0000), connector name( C0 ) [ 34.767836] scsi 0:0:164:0: serial_number(ZA1AGH0L0000C8316HPB) [ 34.773757] scsi 0:0:164:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.800268] mpt3sas_cm0: detecting: handle(0x00ad), sas_address(0x5000c500954319a6), phy(42) [ 34.808704] mpt3sas_cm0: REPORT_LUNS: handle(0x00ad), retries(0) [ 34.824736] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ad), lun(0) [ 34.831834] scsi 0:0:165:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.840507] scsi 0:0:165:0: SSP: handle(0x00ad), sas_addr(0x5000c500954319a6), phy(42), device_name(0x5000c500954319a4) [ 34.851279] scsi 0:0:165:0: enclosure logical id(0x5001636001a0673d), slot(42) [ 34.858585] scsi 0:0:165:0: enclosure level(0x0000), connector name( C0 ) [ 34.865479] scsi 0:0:165:0: serial_number(ZA1AB9SF0000C8316GFV) [ 34.871399] scsi 0:0:165:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.895266] mpt3sas_cm0: detecting: handle(0x00ae), sas_address(0x5000c500954329be), phy(43) [ 34.903704] mpt3sas_cm0: REPORT_LUNS: handle(0x00ae), retries(0) [ 34.918674] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ae), lun(0) [ 34.925782] scsi 0:0:166:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 34.934437] scsi 0:0:166:0: SSP: handle(0x00ae), sas_addr(0x5000c500954329be), phy(43), device_name(0x5000c500954329bc) [ 34.945211] scsi 0:0:166:0: enclosure logical id(0x5001636001a0673d), slot(43) [ 34.952518] scsi 0:0:166:0: enclosure level(0x0000), connector name( C0 ) [ 34.959412] scsi 0:0:166:0: serial_number(ZA1AGGLY0000R829CFMV) [ 34.965331] scsi 0:0:166:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 34.988853] mpt3sas_cm0: detecting: handle(0x00af), sas_address(0x5000c50095432b1a), phy(44) [ 34.997287] mpt3sas_cm0: REPORT_LUNS: handle(0x00af), retries(0) [ 35.014772] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00af), lun(0) [ 35.021879] scsi 0:0:167:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.030533] scsi 0:0:167:0: SSP: handle(0x00af), sas_addr(0x5000c50095432b1a), phy(44), device_name(0x5000c50095432b18) [ 35.041310] scsi 0:0:167:0: enclosure logical id(0x5001636001a0673d), slot(44) [ 35.048615] scsi 0:0:167:0: enclosure level(0x0000), connector name( C0 ) [ 35.055509] scsi 0:0:167:0: serial_number(ZA1AGGJK0000C8315317) [ 35.061428] scsi 0:0:167:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.084262] mpt3sas_cm0: detecting: handle(0x00b0), sas_address(0x5000c500954327aa), phy(45) [ 35.092700] mpt3sas_cm0: REPORT_LUNS: handle(0x00b0), retries(0) [ 35.106645] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b0), lun(0) [ 35.113745] scsi 0:0:168:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.122405] scsi 0:0:168:0: SSP: handle(0x00b0), sas_addr(0x5000c500954327aa), phy(45), device_name(0x5000c500954327a8) [ 35.133180] scsi 0:0:168:0: enclosure logical id(0x5001636001a0673d), slot(45) [ 35.140486] scsi 0:0:168:0: enclosure level(0x0000), connector name( C0 ) [ 35.147377] scsi 0:0:168:0: serial_number(ZA1AGGPV0000C83153P7) [ 35.153298] scsi 0:0:168:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.176266] mpt3sas_cm0: detecting: handle(0x00b1), sas_address(0x5000c5009543229a), phy(46) [ 35.184702] mpt3sas_cm0: REPORT_LUNS: handle(0x00b1), retries(0) [ 35.198934] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b1), lun(0) [ 35.206038] scsi 0:0:169:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.214700] scsi 0:0:169:0: SSP: handle(0x00b1), sas_addr(0x5000c5009543229a), phy(46), device_name(0x5000c50095432298) [ 35.225474] scsi 0:0:169:0: enclosure logical id(0x5001636001a0673d), slot(46) [ 35.232778] scsi 0:0:169:0: enclosure level(0x0000), connector name( C0 ) [ 35.239671] scsi 0:0:169:0: serial_number(ZA1AGGXB0000R704G6EZ) [ 35.245592] scsi 0:0:169:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.271274] mpt3sas_cm0: detecting: handle(0x00b2), sas_address(0x5000c500954328fa), phy(47) [ 35.279729] mpt3sas_cm0: REPORT_LUNS: handle(0x00b2), retries(0) [ 35.292170] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b2), lun(0) [ 35.299280] scsi 0:0:170:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.307941] scsi 0:0:170:0: SSP: handle(0x00b2), sas_addr(0x5000c500954328fa), phy(47), device_name(0x5000c500954328f8) [ 35.318713] scsi 0:0:170:0: enclosure logical id(0x5001636001a0673d), slot(47) [ 35.326019] scsi 0:0:170:0: enclosure level(0x0000), connector name( C0 ) [ 35.332908] scsi 0:0:170:0: serial_number(ZA1AGGMK0000R83026KX) [ 35.338829] scsi 0:0:170:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.362274] mpt3sas_cm0: detecting: handle(0x00bb), sas_address(0x5000c500954cfc86), phy(48) [ 35.370711] mpt3sas_cm0: REPORT_LUNS: handle(0x00bb), retries(0) [ 35.385157] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bb), lun(0) [ 35.392253] scsi 0:0:171:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.400908] scsi 0:0:171:0: SSP: handle(0x00bb), sas_addr(0x5000c500954cfc86), phy(48), device_name(0x5000c500954cfc84) [ 35.411683] scsi 0:0:171:0: enclosure logical id(0x5001636001a0673d), slot(48) [ 35.418987] scsi 0:0:171:0: enclosure level(0x0000), connector name( C0 ) [ 35.425878] scsi 0:0:171:0: serial_number(ZA1ALFE50000R830288F) [ 35.431799] scsi 0:0:171:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.458267] mpt3sas_cm0: detecting: handle(0x00bc), sas_address(0x5000c500954d361e), phy(49) [ 35.466720] mpt3sas_cm0: REPORT_LUNS: handle(0x00bc), retries(0) [ 35.488632] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bc), lun(0) [ 35.495734] scsi 0:0:172:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.504390] scsi 0:0:172:0: SSP: handle(0x00bc), sas_addr(0x5000c500954d361e), phy(49), device_name(0x5000c500954d361c) [ 35.515164] scsi 0:0:172:0: enclosure logical id(0x5001636001a0673d), slot(49) [ 35.522471] scsi 0:0:172:0: enclosure level(0x0000), connector name( C0 ) [ 35.529363] scsi 0:0:172:0: serial_number(ZA1ALTVB0000R8314VBJ) [ 35.535282] scsi 0:0:172:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.564288] mpt3sas_cm0: detecting: handle(0x00bd), sas_address(0x5000c500954cb9ba), phy(50) [ 35.572727] mpt3sas_cm0: REPORT_LUNS: handle(0x00bd), retries(0) [ 35.583980] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bd), lun(0) [ 35.591081] scsi 0:0:173:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.599737] scsi 0:0:173:0: SSP: handle(0x00bd), sas_addr(0x5000c500954cb9ba), phy(50), device_name(0x5000c500954cb9b8) [ 35.610509] scsi 0:0:173:0: enclosure logical id(0x5001636001a0673d), slot(50) [ 35.617815] scsi 0:0:173:0: enclosure level(0x0000), connector name( C0 ) [ 35.624708] scsi 0:0:173:0: serial_number(ZA1ALH520000C8326DUN) [ 35.630627] scsi 0:0:173:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.656277] mpt3sas_cm0: detecting: handle(0x00be), sas_address(0x5000c500954ce226), phy(51) [ 35.664734] mpt3sas_cm0: REPORT_LUNS: handle(0x00be), retries(0) [ 35.680637] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00be), lun(0) [ 35.687732] scsi 0:0:174:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.696392] scsi 0:0:174:0: SSP: handle(0x00be), sas_addr(0x5000c500954ce226), phy(51), device_name(0x5000c500954ce224) [ 35.707164] scsi 0:0:174:0: enclosure logical id(0x5001636001a0673d), slot(51) [ 35.714469] scsi 0:0:174:0: enclosure level(0x0000), connector name( C0 ) [ 35.721363] scsi 0:0:174:0: serial_number(ZA1ALFZF0000C83198RH) [ 35.727282] scsi 0:0:174:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.750279] mpt3sas_cm0: detecting: handle(0x00bf), sas_address(0x5000c500954d03fa), phy(52) [ 35.758719] mpt3sas_cm0: REPORT_LUNS: handle(0x00bf), retries(0) [ 35.777222] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bf), lun(0) [ 35.784335] scsi 0:0:175:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.792995] scsi 0:0:175:0: SSP: handle(0x00bf), sas_addr(0x5000c500954d03fa), phy(52), device_name(0x5000c500954d03f8) [ 35.803764] scsi 0:0:175:0: enclosure logical id(0x5001636001a0673d), slot(52) [ 35.811071] scsi 0:0:175:0: enclosure level(0x0000), connector name( C0 ) [ 35.817961] scsi 0:0:175:0: serial_number(ZA1ALF8W0000R8314REA) [ 35.823881] scsi 0:0:175:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.847283] mpt3sas_cm0: detecting: handle(0x00c0), sas_address(0x5000c500954d0482), phy(53) [ 35.855719] mpt3sas_cm0: REPORT_LUNS: handle(0x00c0), retries(0) [ 35.871097] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c0), lun(0) [ 35.878189] scsi 0:0:176:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.886844] scsi 0:0:176:0: SSP: handle(0x00c0), sas_addr(0x5000c500954d0482), phy(53), device_name(0x5000c500954d0480) [ 35.897618] scsi 0:0:176:0: enclosure logical id(0x5001636001a0673d), slot(53) [ 35.904924] scsi 0:0:176:0: enclosure level(0x0000), connector name( C0 ) [ 35.911815] scsi 0:0:176:0: serial_number(ZA1ALF8F0000R8314T49) [ 35.917737] scsi 0:0:176:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 35.944283] mpt3sas_cm0: detecting: handle(0x00c1), sas_address(0x5000c500954de806), phy(54) [ 35.952720] mpt3sas_cm0: REPORT_LUNS: handle(0x00c1), retries(0) [ 35.966299] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c1), lun(0) [ 35.973405] scsi 0:0:177:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 35.982056] scsi 0:0:177:0: SSP: handle(0x00c1), sas_addr(0x5000c500954de806), phy(54), device_name(0x5000c500954de804) [ 35.992833] scsi 0:0:177:0: enclosure logical id(0x5001636001a0673d), slot(54) [ 36.000137] scsi 0:0:177:0: enclosure level(0x0000), connector name( C0 ) [ 36.007031] scsi 0:0:177:0: serial_number(ZA1AM5L80000R8317PC6) [ 36.012952] scsi 0:0:177:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.036287] mpt3sas_cm0: detecting: handle(0x00c2), sas_address(0x5000c500954e0322), phy(55) [ 36.044726] mpt3sas_cm0: REPORT_LUNS: handle(0x00c2), retries(0) [ 36.062556] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c2), lun(0) [ 36.069688] scsi 0:0:178:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.078351] scsi 0:0:178:0: SSP: handle(0x00c2), sas_addr(0x5000c500954e0322), phy(55), device_name(0x5000c500954e0320) [ 36.089122] scsi 0:0:178:0: enclosure logical id(0x5001636001a0673d), slot(55) [ 36.096429] scsi 0:0:178:0: enclosure level(0x0000), connector name( C0 ) [ 36.103322] scsi 0:0:178:0: serial_number(ZA1AM1Z30000R8317REB) [ 36.109240] scsi 0:0:178:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.132288] mpt3sas_cm0: detecting: handle(0x00c8), sas_address(0x5000c500954d3866), phy(56) [ 36.140729] mpt3sas_cm0: REPORT_LUNS: handle(0x00c8), retries(0) [ 36.162368] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c8), lun(0) [ 36.169473] scsi 0:0:179:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.178136] scsi 0:0:179:0: SSP: handle(0x00c8), sas_addr(0x5000c500954d3866), phy(56), device_name(0x5000c500954d3864) [ 36.188904] scsi 0:0:179:0: enclosure logical id(0x5001636001a0673d), slot(56) [ 36.196211] scsi 0:0:179:0: enclosure level(0x0000), connector name( C0 ) [ 36.203102] scsi 0:0:179:0: serial_number(ZA1ALTQH0000R8303HV0) [ 36.209023] scsi 0:0:179:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.232291] mpt3sas_cm0: detecting: handle(0x00c9), sas_address(0x5000c500954d8792), phy(57) [ 36.240730] mpt3sas_cm0: REPORT_LUNS: handle(0x00c9), retries(0) [ 36.262601] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c9), lun(0) [ 36.269710] scsi 0:0:180:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.278360] scsi 0:0:180:0: SSP: handle(0x00c9), sas_addr(0x5000c500954d8792), phy(57), device_name(0x5000c500954d8790) [ 36.289128] scsi 0:0:180:0: enclosure logical id(0x5001636001a0673d), slot(57) [ 36.296435] scsi 0:0:180:0: enclosure level(0x0000), connector name( C0 ) [ 36.303329] scsi 0:0:180:0: serial_number(ZA1AM40E0000R8301CKS) [ 36.309246] scsi 0:0:180:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.335291] mpt3sas_cm0: detecting: handle(0x00ca), sas_address(0x5000c500954d3d6e), phy(58) [ 36.343741] mpt3sas_cm0: REPORT_LUNS: handle(0x00ca), retries(0) [ 36.365326] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ca), lun(0) [ 36.372443] scsi 0:0:181:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.381105] scsi 0:0:181:0: SSP: handle(0x00ca), sas_addr(0x5000c500954d3d6e), phy(58), device_name(0x5000c500954d3d6c) [ 36.391874] scsi 0:0:181:0: enclosure logical id(0x5001636001a0673d), slot(58) [ 36.399180] scsi 0:0:181:0: enclosure level(0x0000), connector name( C0 ) [ 36.406072] scsi 0:0:181:0: serial_number(ZA1ALTKA0000R8302V05) [ 36.411992] scsi 0:0:181:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.435299] mpt3sas_cm0: detecting: handle(0x00cb), sas_address(0x5000c500954d33ca), phy(59) [ 36.443734] mpt3sas_cm0: REPORT_LUNS: handle(0x00cb), retries(0) [ 36.464080] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00cb), lun(0) [ 36.471172] scsi 0:0:182:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.479823] scsi 0:0:182:0: SSP: handle(0x00cb), sas_addr(0x5000c500954d33ca), phy(59), device_name(0x5000c500954d33c8) [ 36.490600] scsi 0:0:182:0: enclosure logical id(0x5001636001a0673d), slot(59) [ 36.497906] scsi 0:0:182:0: enclosure level(0x0000), connector name( C0 ) [ 36.504800] scsi 0:0:182:0: serial_number(ZA1ALY100000R83028GN) [ 36.510718] scsi 0:0:182:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.535444] mpt3sas_cm0: detecting: handle(0x00ce), sas_address(0x5001636001a0673d), phy(76) [ 36.543894] mpt3sas_cm0: REPORT_LUNS: handle(0x00ce), retries(0) [ 36.550092] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ce), lun(0) [ 36.556575] scsi 0:0:183:0: Enclosure QCT JB4602 SIM 1 1500 PQ: 0 ANSI: 6 [ 36.565046] scsi 0:0:183:0: set ignore_delay_remove for handle(0x00ce) [ 36.571574] scsi 0:0:183:0: SES: handle(0x00ce), sas_addr(0x5001636001a0673d), phy(76), device_name(0x5001636001a0673d) [ 36.582348] scsi 0:0:183:0: enclosure logical id(0x5001636001a0673d), slot(60) [ 36.589652] scsi 0:0:183:0: enclosure level(0x0000), connector name( C0 ) [ 36.596544] scsi 0:0:183:0: serial_number(EO0153300021 ) [ 36.602120] scsi 0:0:183:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.623548] mpt3sas_cm0: expander_add: handle(0x00cf), parent(0x0002), sas_addr(0x500163600190c8ff), phys(79) [ 36.645113] mpt3sas_cm0: detecting: handle(0x00d0), sas_address(0x5000c50095430dd2), phy(0) [ 36.653464] mpt3sas_cm0: REPORT_LUNS: handle(0x00d0), retries(0) [ 36.659996] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d0), lun(0) [ 36.667096] scsi 0:0:184:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.675752] scsi 0:0:184:0: SSP: handle(0x00d0), sas_addr(0x5000c50095430dd2), phy(0), device_name(0x5000c50095430dd0) [ 36.686436] scsi 0:0:184:0: enclosure logical id(0x500163600190c8fd), slot(0) [ 36.693657] scsi 0:0:184:0: enclosure level(0x0000), connector name( C2 ) [ 36.700550] scsi 0:0:184:0: serial_number(ZA1AGHL00000C8316HQ4) [ 36.706468] scsi 0:0:184:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.729297] mpt3sas_cm0: detecting: handle(0x00d1), sas_address(0x5000c50095432246), phy(1) [ 36.737646] mpt3sas_cm0: REPORT_LUNS: handle(0x00d1), retries(0) [ 36.753135] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d1), lun(0) [ 36.760241] scsi 0:0:185:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.768899] scsi 0:0:185:0: SSP: handle(0x00d1), sas_addr(0x5000c50095432246), phy(1), device_name(0x5000c50095432244) [ 36.779589] scsi 0:0:185:0: enclosure logical id(0x500163600190c8fd), slot(1) [ 36.786807] scsi 0:0:185:0: enclosure level(0x0000), connector name( C2 ) [ 36.793699] scsi 0:0:185:0: serial_number(ZA1AGN060000R828A07T) [ 36.799620] scsi 0:0:185:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.822306] mpt3sas_cm0: detecting: handle(0x00d2), sas_address(0x5000c5009569935e), phy(2) [ 36.830659] mpt3sas_cm0: REPORT_LUNS: handle(0x00d2), retries(0) [ 36.851125] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d2), lun(0) [ 36.858241] scsi 0:0:186:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.866901] scsi 0:0:186:0: SSP: handle(0x00d2), sas_addr(0x5000c5009569935e), phy(2), device_name(0x5000c5009569935c) [ 36.877585] scsi 0:0:186:0: enclosure logical id(0x500163600190c8fd), slot(2) [ 36.884803] scsi 0:0:186:0: enclosure level(0x0000), connector name( C2 ) [ 36.891695] scsi 0:0:186:0: serial_number(ZA1AZL5Y0000R832DW4F) [ 36.897616] scsi 0:0:186:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 36.921304] mpt3sas_cm0: detecting: handle(0x00d3), sas_address(0x5000c500a6102e86), phy(3) [ 36.929653] mpt3sas_cm0: REPORT_LUNS: handle(0x00d3), retries(0) [ 36.948862] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d3), lun(0) [ 36.955967] scsi 0:0:187:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 36.964625] scsi 0:0:187:0: SSP: handle(0x00d3), sas_addr(0x5000c500a6102e86), phy(3), device_name(0x5000c500a6102e84) [ 36.975314] scsi 0:0:187:0: enclosure logical id(0x500163600190c8fd), slot(3) [ 36.982532] scsi 0:0:187:0: enclosure level(0x0000), connector name( C2 ) [ 36.989424] scsi 0:0:187:0: serial_number(ZA1B72S80000R8332SVA) [ 36.995345] scsi 0:0:187:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.018316] mpt3sas_cm0: detecting: handle(0x00d4), sas_address(0x5000c50095431c1e), phy(4) [ 37.026672] mpt3sas_cm0: REPORT_LUNS: handle(0x00d4), retries(0) [ 37.039004] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d4), lun(0) [ 37.046119] scsi 0:0:188:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.054782] scsi 0:0:188:0: SSP: handle(0x00d4), sas_addr(0x5000c50095431c1e), phy(4), device_name(0x5000c50095431c1c) [ 37.065467] scsi 0:0:188:0: enclosure logical id(0x500163600190c8fd), slot(4) [ 37.072685] scsi 0:0:188:0: enclosure level(0x0000), connector name( C2 ) [ 37.079579] scsi 0:0:188:0: serial_number(ZA1AGN520000C8316G32) [ 37.085497] scsi 0:0:188:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.108308] mpt3sas_cm0: detecting: handle(0x00d5), sas_address(0x5000c500957facce), phy(5) [ 37.116659] mpt3sas_cm0: REPORT_LUNS: handle(0x00d5), retries(0) [ 37.123211] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d5), lun(0) [ 37.130323] scsi 0:0:189:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.138983] scsi 0:0:189:0: SSP: handle(0x00d5), sas_addr(0x5000c500957facce), phy(5), device_name(0x5000c500957faccc) [ 37.149667] scsi 0:0:189:0: enclosure logical id(0x500163600190c8fd), slot(5) [ 37.156885] scsi 0:0:189:0: enclosure level(0x0000), connector name( C2 ) [ 37.163775] scsi 0:0:189:0: serial_number(ZA1B4F0J0000R833TXB0) [ 37.169696] scsi 0:0:189:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.192313] mpt3sas_cm0: detecting: handle(0x00d6), sas_address(0x5000c5009543265e), phy(6) [ 37.200669] mpt3sas_cm0: REPORT_LUNS: handle(0x00d6), retries(0) [ 37.213038] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d6), lun(0) [ 37.256288] scsi 0:0:190:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.264933] scsi 0:0:190:0: SSP: handle(0x00d6), sas_addr(0x5000c5009543265e), phy(6), device_name(0x5000c5009543265c) [ 37.275622] scsi 0:0:190:0: enclosure logical id(0x500163600190c8fd), slot(6) [ 37.282840] scsi 0:0:190:0: enclosure level(0x0000), connector name( C2 ) [ 37.289732] scsi 0:0:190:0: serial_number(ZA1AGGRH0000R829CU3W) [ 37.295653] scsi 0:0:190:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.318315] mpt3sas_cm0: detecting: handle(0x00d7), sas_address(0x5000c5009543246a), phy(7) [ 37.326666] mpt3sas_cm0: REPORT_LUNS: handle(0x00d7), retries(0) [ 37.333223] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d7), lun(0) [ 37.340334] scsi 0:0:191:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.348998] scsi 0:0:191:0: SSP: handle(0x00d7), sas_addr(0x5000c5009543246a), phy(7), device_name(0x5000c50095432468) [ 37.359681] scsi 0:0:191:0: enclosure logical id(0x500163600190c8fd), slot(7) [ 37.366902] scsi 0:0:191:0: enclosure level(0x0000), connector name( C2 ) [ 37.373793] scsi 0:0:191:0: serial_number(ZA1AGMY10000C8316H8M) [ 37.379713] scsi 0:0:191:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.405316] mpt3sas_cm0: detecting: handle(0x00d8), sas_address(0x5000c5009543205e), phy(8) [ 37.413680] mpt3sas_cm0: REPORT_LUNS: handle(0x00d8), retries(0) [ 37.430233] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d8), lun(0) [ 37.437356] scsi 0:0:192:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.446017] scsi 0:0:192:0: SSP: handle(0x00d8), sas_addr(0x5000c5009543205e), phy(8), device_name(0x5000c5009543205c) [ 37.456700] scsi 0:0:192:0: enclosure logical id(0x500163600190c8fd), slot(8) [ 37.463919] scsi 0:0:192:0: enclosure level(0x0000), connector name( C2 ) [ 37.470811] scsi 0:0:192:0: serial_number(ZA1AGN210000R82892FE) [ 37.476732] scsi 0:0:192:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.501327] mpt3sas_cm0: detecting: handle(0x00d9), sas_address(0x5000c50095432226), phy(9) [ 37.509676] mpt3sas_cm0: REPORT_LUNS: handle(0x00d9), retries(0) [ 37.521923] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d9), lun(0) [ 37.529035] scsi 0:0:193:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.537690] scsi 0:0:193:0: SSP: handle(0x00d9), sas_addr(0x5000c50095432226), phy(9), device_name(0x5000c50095432224) [ 37.548378] scsi 0:0:193:0: enclosure logical id(0x500163600190c8fd), slot(9) [ 37.555596] scsi 0:0:193:0: enclosure level(0x0000), connector name( C2 ) [ 37.562495] scsi 0:0:193:0: serial_number(ZA1AGN0E0000R82893RN) [ 37.568421] scsi 0:0:193:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.594339] mpt3sas_cm0: detecting: handle(0x00da), sas_address(0x5000c5009543121a), phy(10) [ 37.602778] mpt3sas_cm0: REPORT_LUNS: handle(0x00da), retries(0) [ 37.620367] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00da), lun(0) [ 37.627490] scsi 0:0:194:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.636149] scsi 0:0:194:0: SSP: handle(0x00da), sas_addr(0x5000c5009543121a), phy(10), device_name(0x5000c50095431218) [ 37.646921] scsi 0:0:194:0: enclosure logical id(0x500163600190c8fd), slot(10) [ 37.654228] scsi 0:0:194:0: enclosure level(0x0000), connector name( C2 ) [ 37.661118] scsi 0:0:194:0: serial_number(ZA1AGHEZ0000R8289ZB0) [ 37.667040] scsi 0:0:194:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.693338] mpt3sas_cm0: detecting: handle(0x00db), sas_address(0x5000c5009543250e), phy(11) [ 37.701780] mpt3sas_cm0: REPORT_LUNS: handle(0x00db), retries(0) [ 37.722383] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00db), lun(0) [ 37.729504] scsi 0:0:195:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.738165] scsi 0:0:195:0: SSP: handle(0x00db), sas_addr(0x5000c5009543250e), phy(11), device_name(0x5000c5009543250c) [ 37.748939] scsi 0:0:195:0: enclosure logical id(0x500163600190c8fd), slot(11) [ 37.756246] scsi 0:0:195:0: enclosure level(0x0000), connector name( C2 ) [ 37.763135] scsi 0:0:195:0: serial_number(ZA1AGMX90000C8316H8U) [ 37.769058] scsi 0:0:195:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.795332] mpt3sas_cm0: detecting: handle(0x00dc), sas_address(0x5000c500954cf2aa), phy(12) [ 37.803772] mpt3sas_cm0: REPORT_LUNS: handle(0x00dc), retries(0) [ 37.818543] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00dc), lun(0) [ 37.825650] scsi 0:0:196:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.834315] scsi 0:0:196:0: SSP: handle(0x00dc), sas_addr(0x5000c500954cf2aa), phy(12), device_name(0x5000c500954cf2a8) [ 37.845089] scsi 0:0:196:0: enclosure logical id(0x500163600190c8fd), slot(12) [ 37.852396] scsi 0:0:196:0: enclosure level(0x0000), connector name( C2 ) [ 37.859303] scsi 0:0:196:0: serial_number(ZA1ALFL80000R8314RU0) [ 37.865226] scsi 0:0:196:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.891352] mpt3sas_cm0: detecting: handle(0x00dd), sas_address(0x5000c500954d411a), phy(13) [ 37.899793] mpt3sas_cm0: REPORT_LUNS: handle(0x00dd), retries(0) [ 37.921809] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00dd), lun(0) [ 37.928911] scsi 0:0:197:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 37.937569] scsi 0:0:197:0: SSP: handle(0x00dd), sas_addr(0x5000c500954d411a), phy(13), device_name(0x5000c500954d4118) [ 37.948338] scsi 0:0:197:0: enclosure logical id(0x500163600190c8fd), slot(13) [ 37.955644] scsi 0:0:197:0: enclosure level(0x0000), connector name( C2 ) [ 37.962537] scsi 0:0:197:0: serial_number(ZA1ALXPR0000R8302J8K) [ 37.968456] scsi 0:0:197:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 37.991332] mpt3sas_cm0: detecting: handle(0x00de), sas_address(0x5000c500954cb5e2), phy(14) [ 37.999773] mpt3sas_cm0: REPORT_LUNS: handle(0x00de), retries(0) [ 38.018552] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00de), lun(0) [ 38.025673] scsi 0:0:198:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.034329] scsi 0:0:198:0: SSP: handle(0x00de), sas_addr(0x5000c500954cb5e2), phy(14), device_name(0x5000c500954cb5e0) [ 38.045103] scsi 0:0:198:0: enclosure logical id(0x500163600190c8fd), slot(14) [ 38.052409] scsi 0:0:198:0: enclosure level(0x0000), connector name( C2 ) [ 38.059302] scsi 0:0:198:0: serial_number(ZA1ALH7V0000C8326GSW) [ 38.065223] scsi 0:0:198:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.088334] mpt3sas_cm0: detecting: handle(0x00df), sas_address(0x5000c500954df38a), phy(15) [ 38.096772] mpt3sas_cm0: REPORT_LUNS: handle(0x00df), retries(0) [ 38.115116] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00df), lun(0) [ 38.122243] scsi 0:0:199:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.130898] scsi 0:0:199:0: SSP: handle(0x00df), sas_addr(0x5000c500954df38a), phy(15), device_name(0x5000c500954df388) [ 38.141670] scsi 0:0:199:0: enclosure logical id(0x500163600190c8fd), slot(15) [ 38.148976] scsi 0:0:199:0: enclosure level(0x0000), connector name( C2 ) [ 38.155867] scsi 0:0:199:0: serial_number(ZA1AM2550000R8317NB5) [ 38.161790] scsi 0:0:199:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.188340] mpt3sas_cm0: detecting: handle(0x00e0), sas_address(0x5000c500954d8f5e), phy(16) [ 38.196782] mpt3sas_cm0: REPORT_LUNS: handle(0x00e0), retries(0) [ 38.215084] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e0), lun(0) [ 38.222177] scsi 0:0:200:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.230835] scsi 0:0:200:0: SSP: handle(0x00e0), sas_addr(0x5000c500954d8f5e), phy(16), device_name(0x5000c500954d8f5c) [ 38.241609] scsi 0:0:200:0: enclosure logical id(0x500163600190c8fd), slot(16) [ 38.248915] scsi 0:0:200:0: enclosure level(0x0000), connector name( C2 ) [ 38.255807] scsi 0:0:200:0: serial_number(ZA1A3EEM0000R8301QYF) [ 38.261728] scsi 0:0:200:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.287344] mpt3sas_cm0: detecting: handle(0x00e1), sas_address(0x5000c500954ccd9a), phy(17) [ 38.295783] mpt3sas_cm0: REPORT_LUNS: handle(0x00e1), retries(0) [ 38.314433] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e1), lun(0) [ 38.321527] scsi 0:0:201:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.330185] scsi 0:0:201:0: SSP: handle(0x00e1), sas_addr(0x5000c500954ccd9a), phy(17), device_name(0x5000c500954ccd98) [ 38.340957] scsi 0:0:201:0: enclosure logical id(0x500163600190c8fd), slot(17) [ 38.348263] scsi 0:0:201:0: enclosure level(0x0000), connector name( C2 ) [ 38.355154] scsi 0:0:201:0: serial_number(ZA1ALL1N0000R8301RWS) [ 38.361075] scsi 0:0:201:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.384340] mpt3sas_cm0: detecting: handle(0x00e2), sas_address(0x5000c500954d2dbe), phy(18) [ 38.392782] mpt3sas_cm0: REPORT_LUNS: handle(0x00e2), retries(0) [ 38.407181] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e2), lun(0) [ 38.414296] scsi 0:0:202:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.422955] scsi 0:0:202:0: SSP: handle(0x00e2), sas_addr(0x5000c500954d2dbe), phy(18), device_name(0x5000c500954d2dbc) [ 38.433729] scsi 0:0:202:0: enclosure logical id(0x500163600190c8fd), slot(18) [ 38.441035] scsi 0:0:202:0: enclosure level(0x0000), connector name( C2 ) [ 38.447926] scsi 0:0:202:0: serial_number(ZA1ALV6M0000R8314SFK) [ 38.453847] scsi 0:0:202:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.480342] mpt3sas_cm0: detecting: handle(0x00e3), sas_address(0x5000c500954cc1b6), phy(19) [ 38.488793] mpt3sas_cm0: REPORT_LUNS: handle(0x00e3), retries(0) [ 38.511050] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e3), lun(0) [ 38.518157] scsi 0:0:203:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.526817] scsi 0:0:203:0: SSP: handle(0x00e3), sas_addr(0x5000c500954cc1b6), phy(19), device_name(0x5000c500954cc1b4) [ 38.537592] scsi 0:0:203:0: enclosure logical id(0x500163600190c8fd), slot(19) [ 38.544898] scsi 0:0:203:0: enclosure level(0x0000), connector name( C2 ) [ 38.551789] scsi 0:0:203:0: serial_number(ZA1ALGWT0000C8325X34) [ 38.557717] scsi 0:0:203:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.580349] mpt3sas_cm0: detecting: handle(0x00e4), sas_address(0x5000c500954310fe), phy(20) [ 38.588786] mpt3sas_cm0: REPORT_LUNS: handle(0x00e4), retries(0) [ 38.599770] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e4), lun(0) [ 38.606871] scsi 0:0:204:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.615525] scsi 0:0:204:0: SSP: handle(0x00e4), sas_addr(0x5000c500954310fe), phy(20), device_name(0x5000c500954310fc) [ 38.626299] scsi 0:0:204:0: enclosure logical id(0x500163600190c8fd), slot(20) [ 38.633603] scsi 0:0:204:0: enclosure level(0x0000), connector name( C2 ) [ 38.640497] scsi 0:0:204:0: serial_number(ZA1AGHGR0000C8316FM1) [ 38.646417] scsi 0:0:204:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.672357] mpt3sas_cm0: detecting: handle(0x00e5), sas_address(0x5000c500954cc172), phy(21) [ 38.680793] mpt3sas_cm0: REPORT_LUNS: handle(0x00e5), retries(0) [ 38.694322] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e5), lun(0) [ 38.701434] scsi 0:0:205:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.710092] scsi 0:0:205:0: SSP: handle(0x00e5), sas_addr(0x5000c500954cc172), phy(21), device_name(0x5000c500954cc170) [ 38.720863] scsi 0:0:205:0: enclosure logical id(0x500163600190c8fd), slot(21) [ 38.728168] scsi 0:0:205:0: enclosure level(0x0000), connector name( C2 ) [ 38.735061] scsi 0:0:205:0: serial_number(ZA1ALGXA0000C8325WZ8) [ 38.740981] scsi 0:0:205:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.764939] mpt3sas_cm0: detecting: handle(0x00e6), sas_address(0x5000c500954ddfe2), phy(22) [ 38.773382] mpt3sas_cm0: REPORT_LUNS: handle(0x00e6), retries(0) [ 38.785128] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e6), lun(0) [ 38.792258] scsi 0:0:206:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.800921] scsi 0:0:206:0: SSP: handle(0x00e6), sas_addr(0x5000c500954ddfe2), phy(22), device_name(0x5000c500954ddfe0) [ 38.811692] scsi 0:0:206:0: enclosure logical id(0x500163600190c8fd), slot(22) [ 38.818999] scsi 0:0:206:0: enclosure level(0x0000), connector name( C2 ) [ 38.825889] scsi 0:0:206:0: serial_number(ZA1AM5R10000R8317NT7) [ 38.831812] scsi 0:0:206:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.863354] mpt3sas_cm0: detecting: handle(0x00e7), sas_address(0x5000c500954cb456), phy(23) [ 38.871794] mpt3sas_cm0: REPORT_LUNS: handle(0x00e7), retries(0) [ 38.891647] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e7), lun(0) [ 38.898768] scsi 0:0:207:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 38.907429] scsi 0:0:207:0: SSP: handle(0x00e7), sas_addr(0x5000c500954cb456), phy(23), device_name(0x5000c500954cb454) [ 38.918201] scsi 0:0:207:0: enclosure logical id(0x500163600190c8fd), slot(23) [ 38.925507] scsi 0:0:207:0: enclosure level(0x0000), connector name( C2 ) [ 38.932398] scsi 0:0:207:0: serial_number(ZA1ALHA50000C8325WZL) [ 38.938319] scsi 0:0:207:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 38.964357] mpt3sas_cm0: detecting: handle(0x00e8), sas_address(0x5000c500a6104dbe), phy(24) [ 38.972814] mpt3sas_cm0: REPORT_LUNS: handle(0x00e8), retries(0) [ 38.985635] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e8), lun(0) [ 39.028048] scsi 0:0:208:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.036687] scsi 0:0:208:0: SSP: handle(0x00e8), sas_addr(0x5000c500a6104dbe), phy(24), device_name(0x5000c500a6104dbc) [ 39.047459] scsi 0:0:208:0: enclosure logical id(0x500163600190c8fd), slot(24) [ 39.054763] scsi 0:0:208:0: enclosure level(0x0000), connector name( C2 ) [ 39.061656] scsi 0:0:208:0: serial_number(ZA1B729S0000R824UQU0) [ 39.067575] scsi 0:0:208:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.093363] mpt3sas_cm0: detecting: handle(0x00e9), sas_address(0x5000c500954d0c0a), phy(25) [ 39.101805] mpt3sas_cm0: REPORT_LUNS: handle(0x00e9), retries(0) [ 39.115893] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00e9), lun(0) [ 39.123019] scsi 0:0:209:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.131681] scsi 0:0:209:0: SSP: handle(0x00e9), sas_addr(0x5000c500954d0c0a), phy(25), device_name(0x5000c500954d0c08) [ 39.142455] scsi 0:0:209:0: enclosure logical id(0x500163600190c8fd), slot(25) [ 39.149762] scsi 0:0:209:0: enclosure level(0x0000), connector name( C2 ) [ 39.156654] scsi 0:0:209:0: serial_number(ZA1AFAX70000R8314QSP) [ 39.162575] scsi 0:0:209:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.189366] mpt3sas_cm0: detecting: handle(0x00ea), sas_address(0x5000c500954d91a2), phy(26) [ 39.197809] mpt3sas_cm0: REPORT_LUNS: handle(0x00ea), retries(0) [ 39.211918] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ea), lun(0) [ 39.219043] scsi 0:0:210:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.227707] scsi 0:0:210:0: SSP: handle(0x00ea), sas_addr(0x5000c500954d91a2), phy(26), device_name(0x5000c500954d91a0) [ 39.238476] scsi 0:0:210:0: enclosure logical id(0x500163600190c8fd), slot(26) [ 39.245783] scsi 0:0:210:0: enclosure level(0x0000), connector name( C2 ) [ 39.252675] scsi 0:0:210:0: serial_number(ZA1A7W9H0000R8302K3E) [ 39.258595] scsi 0:0:210:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.281360] mpt3sas_cm0: detecting: handle(0x00eb), sas_address(0x5000c500a610243e), phy(27) [ 39.289800] mpt3sas_cm0: REPORT_LUNS: handle(0x00eb), retries(0) [ 39.308644] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00eb), lun(0) [ 39.315765] scsi 0:0:211:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.324427] scsi 0:0:211:0: SSP: handle(0x00eb), sas_addr(0x5000c500a610243e), phy(27), device_name(0x5000c500a610243c) [ 39.335198] scsi 0:0:211:0: enclosure logical id(0x500163600190c8fd), slot(27) [ 39.342504] scsi 0:0:211:0: enclosure level(0x0000), connector name( C2 ) [ 39.349399] scsi 0:0:211:0: serial_number(ZA1B72Z10000R833TUKN) [ 39.355317] scsi 0:0:211:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.378364] mpt3sas_cm0: detecting: handle(0x00ec), sas_address(0x5000c500a6102eb2), phy(28) [ 39.386802] mpt3sas_cm0: REPORT_LUNS: handle(0x00ec), retries(0) [ 39.407510] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ec), lun(0) [ 39.414639] scsi 0:0:212:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.423305] scsi 0:0:212:0: SSP: handle(0x00ec), sas_addr(0x5000c500a6102eb2), phy(28), device_name(0x5000c500a6102eb0) [ 39.434082] scsi 0:0:212:0: enclosure logical id(0x500163600190c8fd), slot(28) [ 39.441385] scsi 0:0:212:0: enclosure level(0x0000), connector name( C2 ) [ 39.448280] scsi 0:0:212:0: serial_number(ZA1B72R60000R83323UR) [ 39.454199] scsi 0:0:212:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.477366] mpt3sas_cm0: detecting: handle(0x00ed), sas_address(0x5000c500a610243a), phy(29) [ 39.485803] mpt3sas_cm0: REPORT_LUNS: handle(0x00ed), retries(0) [ 39.492360] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ed), lun(0) [ 39.499473] scsi 0:0:213:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.508132] scsi 0:0:213:0: SSP: handle(0x00ed), sas_addr(0x5000c500a610243a), phy(29), device_name(0x5000c500a6102438) [ 39.518905] scsi 0:0:213:0: enclosure logical id(0x500163600190c8fd), slot(29) [ 39.526208] scsi 0:0:213:0: enclosure level(0x0000), connector name( C2 ) [ 39.533100] scsi 0:0:213:0: serial_number(ZA1B72ZL0000R833UX7J) [ 39.539022] scsi 0:0:213:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.562372] mpt3sas_cm0: detecting: handle(0x00ee), sas_address(0x5000c500954d9982), phy(30) [ 39.570808] mpt3sas_cm0: REPORT_LUNS: handle(0x00ee), retries(0) [ 39.585762] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ee), lun(0) [ 39.592878] scsi 0:0:214:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.601534] scsi 0:0:214:0: SSP: handle(0x00ee), sas_addr(0x5000c500954d9982), phy(30), device_name(0x5000c500954d9980) [ 39.612307] scsi 0:0:214:0: enclosure logical id(0x500163600190c8fd), slot(30) [ 39.619611] scsi 0:0:214:0: enclosure level(0x0000), connector name( C2 ) [ 39.626505] scsi 0:0:214:0: serial_number(ZA1AM3Q20000R8302N04) [ 39.632426] scsi 0:0:214:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.655370] mpt3sas_cm0: detecting: handle(0x00ef), sas_address(0x5000c500954deaba), phy(31) [ 39.663812] mpt3sas_cm0: REPORT_LUNS: handle(0x00ef), retries(0) [ 39.684103] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ef), lun(0) [ 39.691222] scsi 0:0:215:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.699876] scsi 0:0:215:0: SSP: handle(0x00ef), sas_addr(0x5000c500954deaba), phy(31), device_name(0x5000c500954deab8) [ 39.710651] scsi 0:0:215:0: enclosure logical id(0x500163600190c8fd), slot(31) [ 39.717957] scsi 0:0:215:0: enclosure level(0x0000), connector name( C2 ) [ 39.724848] scsi 0:0:215:0: serial_number(ZA1AJVZZ0000R8317PEX) [ 39.730768] scsi 0:0:215:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.756372] mpt3sas_cm0: detecting: handle(0x00f0), sas_address(0x5000c500954cc8d6), phy(32) [ 39.764821] mpt3sas_cm0: REPORT_LUNS: handle(0x00f0), retries(0) [ 39.780360] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f0), lun(0) [ 39.787476] scsi 0:0:216:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.796130] scsi 0:0:216:0: SSP: handle(0x00f0), sas_addr(0x5000c500954cc8d6), phy(32), device_name(0x5000c500954cc8d4) [ 39.806904] scsi 0:0:216:0: enclosure logical id(0x500163600190c8fd), slot(32) [ 39.814211] scsi 0:0:216:0: enclosure level(0x0000), connector name( C2 ) [ 39.821102] scsi 0:0:216:0: serial_number(ZA1ALGNR0000R8302KGJ) [ 39.827023] scsi 0:0:216:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.850378] mpt3sas_cm0: detecting: handle(0x00f1), sas_address(0x5000c500a6102de2), phy(33) [ 39.858817] mpt3sas_cm0: REPORT_LUNS: handle(0x00f1), retries(0) [ 39.880856] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f1), lun(0) [ 39.887994] scsi 0:0:217:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.896656] scsi 0:0:217:0: SSP: handle(0x00f1), sas_addr(0x5000c500a6102de2), phy(33), device_name(0x5000c500a6102de0) [ 39.907433] scsi 0:0:217:0: enclosure logical id(0x500163600190c8fd), slot(33) [ 39.914737] scsi 0:0:217:0: enclosure level(0x0000), connector name( C2 ) [ 39.921631] scsi 0:0:217:0: serial_number(ZA1B72S70000R833V13M) [ 39.927552] scsi 0:0:217:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 39.950382] mpt3sas_cm0: detecting: handle(0x00f2), sas_address(0x5000c500954cc656), phy(34) [ 39.958825] mpt3sas_cm0: REPORT_LUNS: handle(0x00f2), retries(0) [ 39.978952] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f2), lun(0) [ 39.986066] scsi 0:0:218:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 39.994725] scsi 0:0:218:0: SSP: handle(0x00f2), sas_addr(0x5000c500954cc656), phy(34), device_name(0x5000c500954cc654) [ 40.005499] scsi 0:0:218:0: enclosure logical id(0x500163600190c8fd), slot(34) [ 40.012803] scsi 0:0:218:0: enclosure level(0x0000), connector name( C2 ) [ 40.019697] scsi 0:0:218:0: serial_number(ZA1ALGS20000C8327479) [ 40.025615] scsi 0:0:218:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.048379] mpt3sas_cm0: detecting: handle(0x00f3), sas_address(0x5000c500954d38ca), phy(35) [ 40.056821] mpt3sas_cm0: REPORT_LUNS: handle(0x00f3), retries(0) [ 40.071824] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f3), lun(0) [ 40.078936] scsi 0:0:219:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.087594] scsi 0:0:219:0: SSP: handle(0x00f3), sas_addr(0x5000c500954d38ca), phy(35), device_name(0x5000c500954d38c8) [ 40.098364] scsi 0:0:219:0: enclosure logical id(0x500163600190c8fd), slot(35) [ 40.105670] scsi 0:0:219:0: enclosure level(0x0000), connector name( C2 ) [ 40.112563] scsi 0:0:219:0: serial_number(ZA1ALXWM0000R83026JR) [ 40.118483] scsi 0:0:219:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.144375] mpt3sas_cm0: detecting: handle(0x00f4), sas_address(0x5000c500954d94d2), phy(36) [ 40.152828] mpt3sas_cm0: REPORT_LUNS: handle(0x00f4), retries(0) [ 40.174555] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f4), lun(0) [ 40.181676] scsi 0:0:220:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.190346] scsi 0:0:220:0: SSP: handle(0x00f4), sas_addr(0x5000c500954d94d2), phy(36), device_name(0x5000c500954d94d0) [ 40.201118] scsi 0:0:220:0: enclosure logical id(0x500163600190c8fd), slot(36) [ 40.208424] scsi 0:0:220:0: enclosure level(0x0000), connector name( C2 ) [ 40.215317] scsi 0:0:220:0: serial_number(ZA1AM3VG0000R8313JPX) [ 40.221237] scsi 0:0:220:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.244386] mpt3sas_cm0: detecting: handle(0x00f5), sas_address(0x5000c500a610528a), phy(37) [ 40.252821] mpt3sas_cm0: REPORT_LUNS: handle(0x00f5), retries(0) [ 40.268032] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f5), lun(0) [ 40.275133] scsi 0:0:221:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.283792] scsi 0:0:221:0: SSP: handle(0x00f5), sas_addr(0x5000c500a610528a), phy(37), device_name(0x5000c500a6105288) [ 40.294564] scsi 0:0:221:0: enclosure logical id(0x500163600190c8fd), slot(37) [ 40.301871] scsi 0:0:221:0: enclosure level(0x0000), connector name( C2 ) [ 40.308764] scsi 0:0:221:0: serial_number(ZA1B727K0000R8372HYT) [ 40.314684] scsi 0:0:221:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.337388] mpt3sas_cm0: detecting: handle(0x00f6), sas_address(0x5000c500a6103fda), phy(38) [ 40.345829] mpt3sas_cm0: REPORT_LUNS: handle(0x00f6), retries(0) [ 40.363043] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f6), lun(0) [ 40.370152] scsi 0:0:222:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.378808] scsi 0:0:222:0: SSP: handle(0x00f6), sas_addr(0x5000c500a6103fda), phy(38), device_name(0x5000c500a6103fd8) [ 40.389580] scsi 0:0:222:0: enclosure logical id(0x500163600190c8fd), slot(38) [ 40.396886] scsi 0:0:222:0: enclosure level(0x0000), connector name( C2 ) [ 40.403780] scsi 0:0:222:0: serial_number(ZA1B72HS0000R8331QSS) [ 40.409698] scsi 0:0:222:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.435394] mpt3sas_cm0: detecting: handle(0x00f7), sas_address(0x5000c500a6105756), phy(39) [ 40.443833] mpt3sas_cm0: REPORT_LUNS: handle(0x00f7), retries(0) [ 40.457405] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f7), lun(0) [ 40.464537] scsi 0:0:223:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.473201] scsi 0:0:223:0: SSP: handle(0x00f7), sas_addr(0x5000c500a6105756), phy(39), device_name(0x5000c500a6105754) [ 40.483972] scsi 0:0:223:0: enclosure logical id(0x500163600190c8fd), slot(39) [ 40.491279] scsi 0:0:223:0: enclosure level(0x0000), connector name( C2 ) [ 40.498171] scsi 0:0:223:0: serial_number(ZA1B72670000R833UYPW) [ 40.504089] scsi 0:0:223:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.527388] mpt3sas_cm0: detecting: handle(0x00f8), sas_address(0x5000c500956c75a2), phy(40) [ 40.535824] mpt3sas_cm0: REPORT_LUNS: handle(0x00f8), retries(0) [ 40.557053] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f8), lun(0) [ 40.564179] scsi 0:0:224:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.572841] scsi 0:0:224:0: SSP: handle(0x00f8), sas_addr(0x5000c500956c75a2), phy(40), device_name(0x5000c500956c75a0) [ 40.583616] scsi 0:0:224:0: enclosure logical id(0x500163600190c8fd), slot(40) [ 40.590922] scsi 0:0:224:0: enclosure level(0x0000), connector name( C2 ) [ 40.597814] scsi 0:0:224:0: serial_number(ZA1B0V530000R8332QYC) [ 40.603733] scsi 0:0:224:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.626398] mpt3sas_cm0: detecting: handle(0x00f9), sas_address(0x5000c500a61033e2), phy(41) [ 40.634835] mpt3sas_cm0: REPORT_LUNS: handle(0x00f9), retries(0) [ 40.651950] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00f9), lun(0) [ 40.659063] scsi 0:0:225:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.667720] scsi 0:0:225:0: SSP: handle(0x00f9), sas_addr(0x5000c500a61033e2), phy(41), device_name(0x5000c500a61033e0) [ 40.678492] scsi 0:0:225:0: enclosure logical id(0x500163600190c8fd), slot(41) [ 40.685799] scsi 0:0:225:0: enclosure level(0x0000), connector name( C2 ) [ 40.692691] scsi 0:0:225:0: serial_number(ZA1B737J0000R8332RQ1) [ 40.698612] scsi 0:0:225:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.724390] mpt3sas_cm0: detecting: handle(0x00fa), sas_address(0x5000c500a6104d26), phy(42) [ 40.732846] mpt3sas_cm0: REPORT_LUNS: handle(0x00fa), retries(0) [ 40.747056] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fa), lun(0) [ 40.754174] scsi 0:0:226:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.762829] scsi 0:0:226:0: SSP: handle(0x00fa), sas_addr(0x5000c500a6104d26), phy(42), device_name(0x5000c500a6104d24) [ 40.773604] scsi 0:0:226:0: enclosure logical id(0x500163600190c8fd), slot(42) [ 40.780908] scsi 0:0:226:0: enclosure level(0x0000), connector name( C2 ) [ 40.787800] scsi 0:0:226:0: serial_number(ZA1B72AS0000R833UYB9) [ 40.793721] scsi 0:0:226:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.816402] mpt3sas_cm0: detecting: handle(0x00fb), sas_address(0x5000c500a6102116), phy(43) [ 40.824838] mpt3sas_cm0: REPORT_LUNS: handle(0x00fb), retries(0) [ 40.845682] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fb), lun(0) [ 40.852812] scsi 0:0:227:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.861467] scsi 0:0:227:0: SSP: handle(0x00fb), sas_addr(0x5000c500a6102116), phy(43), device_name(0x5000c500a6102114) [ 40.872241] scsi 0:0:227:0: enclosure logical id(0x500163600190c8fd), slot(43) [ 40.879547] scsi 0:0:227:0: enclosure level(0x0000), connector name( C2 ) [ 40.886439] scsi 0:0:227:0: serial_number(ZA1B731E0000R833UYGB) [ 40.892361] scsi 0:0:227:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 40.915989] mpt3sas_cm0: detecting: handle(0x00fc), sas_address(0x5000c500a610647a), phy(44) [ 40.924429] mpt3sas_cm0: REPORT_LUNS: handle(0x00fc), retries(0) [ 40.930969] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fc), lun(0) [ 40.938081] scsi 0:0:228:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 40.946742] scsi 0:0:228:0: SSP: handle(0x00fc), sas_addr(0x5000c500a610647a), phy(44), device_name(0x5000c500a6106478) [ 40.957516] scsi 0:0:228:0: enclosure logical id(0x500163600190c8fd), slot(44) [ 40.964820] scsi 0:0:228:0: enclosure level(0x0000), connector name( C2 ) [ 40.971712] scsi 0:0:228:0: serial_number(ZA1B72310000R8314SBW) [ 40.977632] scsi 0:0:228:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.000409] mpt3sas_cm0: detecting: handle(0x00fd), sas_address(0x5000c500a610264e), phy(45) [ 41.008845] mpt3sas_cm0: REPORT_LUNS: handle(0x00fd), retries(0) [ 41.027399] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fd), lun(0) [ 41.034515] scsi 0:0:229:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.043193] scsi 0:0:229:0: SSP: handle(0x00fd), sas_addr(0x5000c500a610264e), phy(45), device_name(0x5000c500a610264c) [ 41.053967] scsi 0:0:229:0: enclosure logical id(0x500163600190c8fd), slot(45) [ 41.061274] scsi 0:0:229:0: enclosure level(0x0000), connector name( C2 ) [ 41.068166] scsi 0:0:229:0: serial_number(ZA1B72XL0000R833UYDL) [ 41.074087] scsi 0:0:229:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.103405] mpt3sas_cm0: detecting: handle(0x00fe), sas_address(0x5000c500954d4402), phy(46) [ 41.111859] mpt3sas_cm0: REPORT_LUNS: handle(0x00fe), retries(0) [ 41.126361] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00fe), lun(0) [ 41.133488] scsi 0:0:230:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.142148] scsi 0:0:230:0: SSP: handle(0x00fe), sas_addr(0x5000c500954d4402), phy(46), device_name(0x5000c500954d4400) [ 41.152919] scsi 0:0:230:0: enclosure logical id(0x500163600190c8fd), slot(46) [ 41.160224] scsi 0:0:230:0: enclosure level(0x0000), connector name( C2 ) [ 41.167117] scsi 0:0:230:0: serial_number(ZA1AHBRF0000R8318PP8) [ 41.173038] scsi 0:0:230:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.196404] mpt3sas_cm0: detecting: handle(0x00ff), sas_address(0x5000c500957eb4c6), phy(47) [ 41.204840] mpt3sas_cm0: REPORT_LUNS: handle(0x00ff), retries(0) [ 41.222649] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ff), lun(0) [ 41.229796] scsi 0:0:231:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.238456] scsi 0:0:231:0: SSP: handle(0x00ff), sas_addr(0x5000c500957eb4c6), phy(47), device_name(0x5000c500957eb4c4) [ 41.249226] scsi 0:0:231:0: enclosure logical id(0x500163600190c8fd), slot(47) [ 41.256530] scsi 0:0:231:0: enclosure level(0x0000), connector name( C2 ) [ 41.263423] scsi 0:0:231:0: serial_number(ZA1B6AXY0000R8332SL8) [ 41.269344] scsi 0:0:231:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.292410] mpt3sas_cm0: detecting: handle(0x0100), sas_address(0x5000c500954328f6), phy(48) [ 41.300852] mpt3sas_cm0: REPORT_LUNS: handle(0x0100), retries(0) [ 41.318270] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0100), lun(0) [ 41.325387] scsi 0:0:232:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.334046] scsi 0:0:232:0: SSP: handle(0x0100), sas_addr(0x5000c500954328f6), phy(48), device_name(0x5000c500954328f4) [ 41.344822] scsi 0:0:232:0: enclosure logical id(0x500163600190c8fd), slot(48) [ 41.352127] scsi 0:0:232:0: enclosure level(0x0000), connector name( C2 ) [ 41.359019] scsi 0:0:232:0: serial_number(ZA1AGMWA0000C8316H64) [ 41.364940] scsi 0:0:232:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.388408] mpt3sas_cm0: detecting: handle(0x0101), sas_address(0x5000c50095432166), phy(49) [ 41.396845] mpt3sas_cm0: REPORT_LUNS: handle(0x0101), retries(0) [ 41.415451] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0101), lun(0) [ 41.422579] scsi 0:0:233:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.431254] scsi 0:0:233:0: SSP: handle(0x0101), sas_addr(0x5000c50095432166), phy(49), device_name(0x5000c50095432164) [ 41.442031] scsi 0:0:233:0: enclosure logical id(0x500163600190c8fd), slot(49) [ 41.449335] scsi 0:0:233:0: enclosure level(0x0000), connector name( C2 ) [ 41.456227] scsi 0:0:233:0: serial_number(ZA1AGGY80000C724RXCM) [ 41.462149] scsi 0:0:233:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.488418] mpt3sas_cm0: detecting: handle(0x0102), sas_address(0x5000c500954313be), phy(50) [ 41.496855] mpt3sas_cm0: REPORT_LUNS: handle(0x0102), retries(0) [ 41.511694] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0102), lun(0) [ 41.518835] scsi 0:0:234:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.527496] scsi 0:0:234:0: SSP: handle(0x0102), sas_addr(0x5000c500954313be), phy(50), device_name(0x5000c500954313bc) [ 41.538268] scsi 0:0:234:0: enclosure logical id(0x500163600190c8fd), slot(50) [ 41.545572] scsi 0:0:234:0: enclosure level(0x0000), connector name( C2 ) [ 41.552466] scsi 0:0:234:0: serial_number(ZA1AGHD30000C8316HPY) [ 41.558389] scsi 0:0:234:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.581417] mpt3sas_cm0: detecting: handle(0x0103), sas_address(0x5000c50095431582), phy(51) [ 41.589859] mpt3sas_cm0: REPORT_LUNS: handle(0x0103), retries(0) [ 41.596411] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0103), lun(0) [ 41.603532] scsi 0:0:235:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.612191] scsi 0:0:235:0: SSP: handle(0x0103), sas_addr(0x5000c50095431582), phy(51), device_name(0x5000c50095431580) [ 41.622960] scsi 0:0:235:0: enclosure logical id(0x500163600190c8fd), slot(51) [ 41.630267] scsi 0:0:235:0: enclosure level(0x0000), connector name( C2 ) [ 41.637157] scsi 0:0:235:0: serial_number(ZA1AGHB90000R7245BUM) [ 41.643078] scsi 0:0:235:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.666416] mpt3sas_cm0: detecting: handle(0x0104), sas_address(0x5000c50095430f2a), phy(52) [ 41.674855] mpt3sas_cm0: REPORT_LUNS: handle(0x0104), retries(0) [ 41.693981] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0104), lun(0) [ 41.701110] scsi 0:0:236:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.709769] scsi 0:0:236:0: SSP: handle(0x0104), sas_addr(0x5000c50095430f2a), phy(52), device_name(0x5000c50095430f28) [ 41.720541] scsi 0:0:236:0: enclosure logical id(0x500163600190c8fd), slot(52) [ 41.727847] scsi 0:0:236:0: enclosure level(0x0000), connector name( C2 ) [ 41.734740] scsi 0:0:236:0: serial_number(ZA1AGHJF0000C8316HQY) [ 41.740658] scsi 0:0:236:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.766426] mpt3sas_cm0: detecting: handle(0x0105), sas_address(0x5000c500954327f2), phy(53) [ 41.774862] mpt3sas_cm0: REPORT_LUNS: handle(0x0105), retries(0) [ 41.793436] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0105), lun(0) [ 41.800563] scsi 0:0:237:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.809225] scsi 0:0:237:0: SSP: handle(0x0105), sas_addr(0x5000c500954327f2), phy(53), device_name(0x5000c500954327f0) [ 41.820001] scsi 0:0:237:0: enclosure logical id(0x500163600190c8fd), slot(53) [ 41.827308] scsi 0:0:237:0: enclosure level(0x0000), connector name( C2 ) [ 41.834198] scsi 0:0:237:0: serial_number(ZA1AGGP90000R814TJY2) [ 41.840119] scsi 0:0:237:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.863427] mpt3sas_cm0: detecting: handle(0x0106), sas_address(0x5000c500954327fa), phy(54) [ 41.871863] mpt3sas_cm0: REPORT_LUNS: handle(0x0106), retries(0) [ 41.893826] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0106), lun(0) [ 41.900957] scsi 0:0:238:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 41.909614] scsi 0:0:238:0: SSP: handle(0x0106), sas_addr(0x5000c500954327fa), phy(54), device_name(0x5000c500954327f8) [ 41.920391] scsi 0:0:238:0: enclosure logical id(0x500163600190c8fd), slot(54) [ 41.927697] scsi 0:0:238:0: enclosure level(0x0000), connector name( C2 ) [ 41.934589] scsi 0:0:238:0: serial_number(ZA1AGGP70000C8316FPD) [ 41.940510] scsi 0:0:238:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 41.963423] mpt3sas_cm0: detecting: handle(0x0107), sas_address(0x5000c5009543222e), phy(55) [ 41.971860] mpt3sas_cm0: REPORT_LUNS: handle(0x0107), retries(0) [ 41.985229] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0107), lun(0) [ 41.992350] scsi 0:0:239:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 42.001009] scsi 0:0:239:0: SSP: handle(0x0107), sas_addr(0x5000c5009543222e), phy(55), device_name(0x5000c5009543222c) [ 42.011783] scsi 0:0:239:0: enclosure logical id(0x500163600190c8fd), slot(55) [ 42.019089] scsi 0:0:239:0: enclosure level(0x0000), connector name( C2 ) [ 42.025983] scsi 0:0:239:0: serial_number(ZA1AGH6Q0000C8316FPY) [ 42.031902] scsi 0:0:239:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 42.058424] mpt3sas_cm0: detecting: handle(0x0108), sas_address(0x5000c50095372626), phy(56) [ 42.066872] mpt3sas_cm0: REPORT_LUNS: handle(0x0108), retries(0) [ 42.076223] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0108), lun(0) [ 42.083345] scsi 0:0:240:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 42.092021] scsi 0:0:240:0: SSP: handle(0x0108), sas_addr(0x5000c50095372626), phy(56), device_name(0x5000c50095372624) [ 42.102795] scsi 0:0:240:0: enclosure logical id(0x500163600190c8fd), slot(56) [ 42.110101] scsi 0:0:240:0: enclosure level(0x0000), connector name( C2 ) [ 42.116993] scsi 0:0:240:0: serial_number(ZA1ABZ4Y0000R829D6VF) [ 42.122913] scsi 0:0:240:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 42.146431] mpt3sas_cm0: detecting: handle(0x0109), sas_address(0x5000c50095430cb2), phy(57) [ 42.154869] mpt3sas_cm0: REPORT_LUNS: handle(0x0109), retries(0) [ 42.175188] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0109), lun(0) [ 42.182331] scsi 0:0:241:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 42.190991] scsi 0:0:241:0: SSP: handle(0x0109), sas_addr(0x5000c50095430cb2), phy(57), device_name(0x5000c50095430cb0) [ 42.201763] scsi 0:0:241:0: enclosure logical id(0x500163600190c8fd), slot(57) [ 42.209067] scsi 0:0:241:0: enclosure level(0x0000), connector name( C2 ) [ 42.215958] scsi 0:0:241:0: serial_number(ZA1AGHLV0000R823QBK4) [ 42.221881] scsi 0:0:241:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 42.245434] mpt3sas_cm0: detecting: handle(0x010a), sas_address(0x5000c50095430c32), phy(58) [ 42.253873] mpt3sas_cm0: REPORT_LUNS: handle(0x010a), retries(0) [ 42.275009] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010a), lun(0) [ 42.282132] scsi 0:0:242:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 42.290785] scsi 0:0:242:0: SSP: handle(0x010a), sas_addr(0x5000c50095430c32), phy(58), device_name(0x5000c50095430c30) [ 42.301562] scsi 0:0:242:0: enclosure logical id(0x500163600190c8fd), slot(58) [ 42.308868] scsi 0:0:242:0: enclosure level(0x0000), connector name( C2 ) [ 42.315760] scsi 0:0:242:0: serial_number(ZA1AGHMK0000R824E79X) [ 42.321678] scsi 0:0:242:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 42.344428] mpt3sas_cm0: detecting: handle(0x010b), sas_address(0x5000c500954327ce), phy(59) [ 42.352866] mpt3sas_cm0: REPORT_LUNS: handle(0x010b), retries(0) [ 42.373829] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010b), lun(0) [ 42.380946] scsi 0:0:243:0: Direct-Access SEAGATE ST8000NM0075 E004 PQ: 0 ANSI: 6 [ 42.389602] scsi 0:0:243:0: SSP: handle(0x010b), sas_addr(0x5000c500954327ce), phy(59), device_name(0x5000c500954327cc) [ 42.400374] scsi 0:0:243:0: enclosure logical id(0x500163600190c8fd), slot(59) [ 42.407678] scsi 0:0:243:0: enclosure level(0x0000), connector name( C2 ) [ 42.414571] scsi 0:0:243:0: serial_number(ZA1AGGPH0000R829CU4W) [ 42.420491] scsi 0:0:243:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 42.444591] mpt3sas_cm0: detecting: handle(0x010d), sas_address(0x500163600190c8fd), phy(76) [ 42.453045] mpt3sas_cm0: REPORT_LUNS: handle(0x010d), retries(0) [ 42.459236] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010d), lun(0) [ 42.465741] scsi 0:0:244:0: Enclosure QCT JB4602 SIM 1 1500 PQ: 0 ANSI: 6 [ 42.474208] scsi 0:0:244:0: set ignore_delay_remove for handle(0x010d) [ 42.480734] scsi 0:0:244:0: SES: handle(0x010d), sas_addr(0x500163600190c8fd), phy(76), device_name(0x500163600190c8fd) [ 42.491505] scsi 0:0:244:0: enclosure logical id(0x500163600190c8fd), slot(60) [ 42.498811] scsi 0:0:244:0: enclosure level(0x0000), connector name( C2 ) [ 42.505704] scsi 0:0:244:0: serial_number(EO0152000010 ) [ 42.511277] scsi 0:0:244:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 42.539984] mpt3sas_cm0: port enable: SUCCESS [ 43.014021] sd 0:0:3:0: [sdc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.014032] sd 0:0:15:0: [sdo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.014035] sd 0:0:15:0: [sdo] 4096-byte physical blocks [ 43.014587] sd 0:0:31:0: [sdae] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.014589] sd 0:0:31:0: [sdae] 4096-byte physical blocks [ 43.015313] sd 0:0:39:0: [sdam] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.015316] sd 0:0:39:0: [sdam] 4096-byte physical blocks [ 43.015866] sd 0:0:49:0: [sdaw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.015868] sd 0:0:49:0: [sdaw] 4096-byte physical blocks [ 43.015952] sd 0:0:1:0: [sda] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.015953] sd 0:0:1:0: [sda] 4096-byte physical blocks [ 43.016017] sd 0:0:51:0: [sday] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.016019] sd 0:0:51:0: [sday] 4096-byte physical blocks [ 43.016182] sd 0:0:8:0: [sdh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.016183] sd 0:0:8:0: [sdh] 4096-byte physical blocks [ 43.016227] sd 0:0:12:0: [sdl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.016229] sd 0:0:12:0: [sdl] 4096-byte physical blocks [ 43.017253] sd 0:0:13:0: [sdm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.017256] sd 0:0:13:0: [sdm] 4096-byte physical blocks [ 43.017642] sd 0:0:15:0: [sdo] Write Protect is off [ 43.017644] sd 0:0:15:0: [sdo] Mode Sense: db 00 10 08 [ 43.017693] sd 0:0:9:0: [sdi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.017695] sd 0:0:9:0: [sdi] 4096-byte physical blocks [ 43.017710] sd 0:0:39:0: [sdam] Write Protect is off [ 43.017712] sd 0:0:39:0: [sdam] Mode Sense: db 00 10 08 [ 43.018232] sd 0:0:49:0: [sdaw] Write Protect is off [ 43.018234] sd 0:0:49:0: [sdaw] Mode Sense: db 00 10 08 [ 43.018344] sd 0:0:1:0: [sda] Write Protect is off [ 43.018345] sd 0:0:1:0: [sda] Mode Sense: db 00 10 08 [ 43.018417] sd 0:0:51:0: [sday] Write Protect is off [ 43.018418] sd 0:0:51:0: [sday] Mode Sense: db 00 10 08 [ 43.018534] sd 0:0:8:0: [sdh] Write Protect is off [ 43.018535] sd 0:0:8:0: [sdh] Mode Sense: db 00 10 08 [ 43.018595] sd 0:0:12:0: [sdl] Write Protect is off [ 43.018596] sd 0:0:12:0: [sdl] Mode Sense: db 00 10 08 [ 43.018707] sd 0:0:2:0: [sdb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.018711] sd 0:0:2:0: [sdb] 4096-byte physical blocks [ 43.019008] sd 0:0:15:0: [sdo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.019026] sd 0:0:31:0: [sdae] Write Protect is off [ 43.019027] sd 0:0:31:0: [sdae] Mode Sense: db 00 10 08 [ 43.019332] sd 0:0:38:0: [sdal] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.019334] sd 0:0:38:0: [sdal] 4096-byte physical blocks [ 43.019546] sd 0:0:49:0: [sdaw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.019583] sd 0:0:98:0: [sdcs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.019584] sd 0:0:98:0: [sdcs] 4096-byte physical blocks [ 43.019650] sd 0:0:1:0: [sda] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.019716] sd 0:0:51:0: [sday] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.019821] sd 0:0:8:0: [sdh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.019836] sd 0:0:6:0: [sdf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.019837] sd 0:0:20:0: [sdt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.019837] sd 0:0:6:0: [sdf] 4096-byte physical blocks [ 43.019838] sd 0:0:20:0: [sdt] 4096-byte physical blocks [ 43.019885] sd 0:0:12:0: [sdl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.019943] sd 0:0:13:0: [sdm] Write Protect is off [ 43.019945] sd 0:0:13:0: [sdm] Mode Sense: db 00 10 08 [ 43.019980] sd 0:0:45:0: [sdas] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.019981] sd 0:0:45:0: [sdas] 4096-byte physical blocks [ 43.020155] sd 0:0:43:0: [sdaq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020157] sd 0:0:43:0: [sdaq] 4096-byte physical blocks [ 43.020187] sd 0:0:27:0: [sdaa] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020188] sd 0:0:27:0: [sdaa] 4096-byte physical blocks [ 43.020208] sd 0:0:101:0: [sdcv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020209] sd 0:0:101:0: [sdcv] 4096-byte physical blocks [ 43.020382] sd 0:0:7:0: [sdg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020383] sd 0:0:7:0: [sdg] 4096-byte physical blocks [ 43.020424] sd 0:0:35:0: [sdai] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020426] sd 0:0:35:0: [sdai] 4096-byte physical blocks [ 43.020574] sd 0:0:5:0: [sde] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020576] sd 0:0:5:0: [sde] 4096-byte physical blocks [ 43.020702] sd 0:0:48:0: [sdav] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020704] sd 0:0:48:0: [sdav] 4096-byte physical blocks [ 43.020725] sd 0:0:9:0: [sdi] Write Protect is off [ 43.020727] sd 0:0:9:0: [sdi] Mode Sense: db 00 10 08 [ 43.020813] sd 0:0:31:0: [sdae] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.020898] sd 0:0:108:0: [sddc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020900] sd 0:0:108:0: [sddc] 4096-byte physical blocks [ 43.020981] sd 0:0:22:0: [sdv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.020983] sd 0:0:22:0: [sdv] 4096-byte physical blocks [ 43.021073] sd 0:0:14:0: [sdn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.021075] sd 0:0:14:0: [sdn] 4096-byte physical blocks [ 43.021089] sd 0:0:11:0: [sdk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.021091] sd 0:0:11:0: [sdk] 4096-byte physical blocks [ 43.021259] sd 0:0:2:0: [sdb] Write Protect is off [ 43.021260] sd 0:0:13:0: [sdm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.021261] sd 0:0:2:0: [sdb] Mode Sense: db 00 10 08 [ 43.021404] sd 0:0:25:0: [sdy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.021406] sd 0:0:25:0: [sdy] 4096-byte physical blocks [ 43.021829] sd 0:0:38:0: [sdal] Write Protect is off [ 43.021831] sd 0:0:38:0: [sdal] Mode Sense: db 00 10 08 [ 43.022004] sd 0:0:10:0: [sdj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.022006] sd 0:0:10:0: [sdj] 4096-byte physical blocks [ 43.022086] sd 0:0:23:0: [sdw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.022090] sd 0:0:23:0: [sdw] 4096-byte physical blocks [ 43.022134] sd 0:0:30:0: [sdad] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.022135] sd 0:0:30:0: [sdad] 4096-byte physical blocks [ 43.022167] sd 0:0:19:0: [sds] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.022169] sd 0:0:19:0: [sds] 4096-byte physical blocks [ 43.022216] sd 0:0:20:0: [sdt] Write Protect is off [ 43.022217] sd 0:0:20:0: [sdt] Mode Sense: db 00 10 08 [ 43.022270] sd 0:0:29:0: [sdac] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.022272] sd 0:0:29:0: [sdac] 4096-byte physical blocks [ 43.022427] sd 0:0:6:0: [sdf] Write Protect is off [ 43.022429] sd 0:0:6:0: [sdf] Mode Sense: db 00 10 08 [ 43.022572] sd 0:0:27:0: [sdaa] Write Protect is off [ 43.022573] sd 0:0:27:0: [sdaa] Mode Sense: db 00 10 08 [ 43.022621] sd 0:0:2:0: [sdb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.022722] sd 0:0:39:0: [sdam] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.022797] sd 0:0:43:0: [sdaq] Write Protect is off [ 43.022800] sd 0:0:43:0: [sdaq] Mode Sense: db 00 10 08 [ 43.022826] sd 0:0:35:0: [sdai] Write Protect is off [ 43.022827] sd 0:0:35:0: [sdai] Mode Sense: db 00 10 08 [ 43.022845] sd 0:0:7:0: [sdg] Write Protect is off [ 43.022847] sd 0:0:7:0: [sdg] Mode Sense: db 00 10 08 [ 43.022856] sd 0:0:45:0: [sdas] Write Protect is off [ 43.022857] sd 0:0:45:0: [sdas] Mode Sense: db 00 10 08 [ 43.022916] sd 0:0:46:0: [sdat] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.022918] sd 0:0:46:0: [sdat] 4096-byte physical blocks [ 43.022926] sd 0:0:5:0: [sde] Write Protect is off [ 43.022929] sd 0:0:5:0: [sde] Mode Sense: db 00 10 08 [ 43.023023] sd 0:0:57:0: [sdbe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023027] sd 0:0:57:0: [sdbe] 4096-byte physical blocks [ 43.023059] sd 0:0:98:0: [sdcs] Write Protect is off [ 43.023061] sd 0:0:98:0: [sdcs] Mode Sense: db 00 10 08 [ 43.023284] sd 0:0:87:0: [sdch] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023286] sd 0:0:87:0: [sdch] 4096-byte physical blocks [ 43.023306] sd 0:0:108:0: [sddc] Write Protect is off [ 43.023308] sd 0:0:108:0: [sddc] Mode Sense: db 00 10 08 [ 43.023334] sd 0:0:123:0: [sddq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023336] sd 0:0:123:0: [sddq] 4096-byte physical blocks [ 43.023350] sd 0:0:40:0: [sdan] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023352] sd 0:0:40:0: [sdan] 4096-byte physical blocks [ 43.023372] sd 0:0:14:0: [sdn] Write Protect is off [ 43.023373] sd 0:0:14:0: [sdn] Mode Sense: db 00 10 08 [ 43.023378] sd 0:0:92:0: [sdcm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023380] sd 0:0:92:0: [sdcm] 4096-byte physical blocks [ 43.023382] sd 0:0:22:0: [sdv] Write Protect is off [ 43.023384] sd 0:0:22:0: [sdv] Mode Sense: db 00 10 08 [ 43.023431] sd 0:0:38:0: [sdal] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.023440] sd 0:0:11:0: [sdk] Write Protect is off [ 43.023441] sd 0:0:11:0: [sdk] Mode Sense: db 00 10 08 [ 43.023503] sd 0:0:20:0: [sdt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.023547] sd 0:0:33:0: [sdag] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023548] sd 0:0:33:0: [sdag] 4096-byte physical blocks [ 43.023633] sd 0:0:48:0: [sdav] Write Protect is off [ 43.023635] sd 0:0:48:0: [sdav] Mode Sense: db 00 10 08 [ 43.023639] sd 0:0:34:0: [sdah] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023643] sd 0:0:34:0: [sdah] 4096-byte physical blocks [ 43.023655] sd 0:0:101:0: [sdcv] Write Protect is off [ 43.023657] sd 0:0:101:0: [sdcv] Mode Sense: db 00 10 08 [ 43.023680] sd 0:0:59:0: [sdbg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023683] sd 0:0:59:0: [sdbg] 4096-byte physical blocks [ 43.023727] sd 0:0:4:0: [sdd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.023729] sd 0:0:4:0: [sdd] 4096-byte physical blocks [ 43.023779] sd 0:0:6:0: [sdf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.023871] sd 0:0:27:0: [sdaa] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024053] sd 0:0:125:0: [sdds] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024055] sd 0:0:125:0: [sdds] 4096-byte physical blocks [ 43.024111] sd 0:0:43:0: [sdaq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024127] sd 0:0:35:0: [sdai] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024138] sd 0:0:25:0: [sdy] Write Protect is off [ 43.024139] sd 0:0:25:0: [sdy] Mode Sense: db 00 10 08 [ 43.024153] sd 0:0:7:0: [sdg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024187] sd 0:0:45:0: [sdas] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024192] sd 0:0:41:0: [sdao] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024193] sd 0:0:41:0: [sdao] 4096-byte physical blocks [ 43.024226] sd 0:0:5:0: [sde] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024294] sd 0:0:124:0: [sddr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024295] sd 0:0:124:0: [sddr] 4096-byte physical blocks [ 43.024324] sd 0:0:10:0: [sdj] Write Protect is off [ 43.024325] sd 0:0:10:0: [sdj] Mode Sense: db 00 10 08 [ 43.024437] sd 0:0:98:0: [sdcs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024443] sd 0:0:16:0: [sdp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024444] sd 0:0:16:0: [sdp] 4096-byte physical blocks [ 43.024457] sd 0:0:23:0: [sdw] Write Protect is off [ 43.024458] sd 0:0:23:0: [sdw] Mode Sense: db 00 10 08 [ 43.024473] sd 0:0:30:0: [sdad] Write Protect is off [ 43.024475] sd 0:0:30:0: [sdad] Mode Sense: db 00 10 08 [ 43.024499] sd 0:0:50:0: [sdax] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024502] sd 0:0:50:0: [sdax] 4096-byte physical blocks [ 43.024510] sd 0:0:126:0: [sddt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024511] sd 0:0:126:0: [sddt] 4096-byte physical blocks [ 43.024526] sd 0:0:71:0: [sdbr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024527] sd 0:0:71:0: [sdbr] 4096-byte physical blocks [ 43.024538] sd 0:0:79:0: [sdbz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024539] sd 0:0:79:0: [sdbz] 4096-byte physical blocks [ 43.024547] sd 0:0:91:0: [sdcl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024548] sd 0:0:91:0: [sdcl] 4096-byte physical blocks [ 43.024552] sd 0:0:19:0: [sds] Write Protect is off [ 43.024553] sd 0:0:19:0: [sds] Mode Sense: db 00 10 08 [ 43.024613] sd 0:0:108:0: [sddc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024635] sd 0:0:29:0: [sdac] Write Protect is off [ 43.024636] sd 0:0:76:0: [sdbw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024637] sd 0:0:29:0: [sdac] Mode Sense: db 00 10 08 [ 43.024637] sd 0:0:76:0: [sdbw] 4096-byte physical blocks [ 43.024659] sd 0:0:36:0: [sdaj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024660] sd 0:0:36:0: [sdaj] 4096-byte physical blocks [ 43.024685] sd 0:0:68:0: [sdbo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024686] sd 0:0:68:0: [sdbo] 4096-byte physical blocks [ 43.024691] sd 0:0:14:0: [sdn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024732] sd 0:0:84:0: [sdce] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024734] sd 0:0:84:0: [sdce] 4096-byte physical blocks [ 43.024741] sd 0:0:131:0: [sddy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024742] sd 0:0:131:0: [sddy] 4096-byte physical blocks [ 43.024764] sd 0:0:75:0: [sdbv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024766] sd 0:0:75:0: [sdbv] 4096-byte physical blocks [ 43.024772] sd 0:0:22:0: [sdv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024774] sd 0:0:11:0: [sdk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.024869] sd 0:0:106:0: [sdda] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024871] sd 0:0:106:0: [sdda] 4096-byte physical blocks [ 43.024917] sd 0:0:128:0: [sddv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024918] sd 0:0:128:0: [sddv] 4096-byte physical blocks [ 43.024931] sd 0:0:129:0: [sddw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024933] sd 0:0:129:0: [sddw] 4096-byte physical blocks [ 43.024953] sd 0:0:88:0: [sdci] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.024955] sd 0:0:88:0: [sdci] 4096-byte physical blocks [ 43.025009] sd 0:0:48:0: [sdav] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.025026] sd 0:0:101:0: [sdcv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.025064] sd 0:0:127:0: [sddu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025065] sd 0:0:127:0: [sddu] 4096-byte physical blocks [ 43.025104] sd 0:0:97:0: [sdcr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025105] sd 0:0:97:0: [sdcr] 4096-byte physical blocks [ 43.025115] sd 0:0:78:0: [sdby] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025116] sd 0:0:78:0: [sdby] 4096-byte physical blocks [ 43.025137] sd 0:0:100:0: [sdcu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025138] sd 0:0:100:0: [sdcu] 4096-byte physical blocks [ 43.025197] sd 0:0:70:0: [sdbq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025198] sd 0:0:70:0: [sdbq] 4096-byte physical blocks [ 43.025322] sd 0:0:62:0: [sdbi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025323] sd 0:0:62:0: [sdbi] 4096-byte physical blocks [ 43.025333] sd 0:0:83:0: [sdcd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025334] sd 0:0:83:0: [sdcd] 4096-byte physical blocks [ 43.025373] sd 0:0:56:0: [sdbd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025374] sd 0:0:56:0: [sdbd] 4096-byte physical blocks [ 43.025431] sd 0:0:9:0: [sdi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.025449] sd 0:0:132:0: [sddz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025451] sd 0:0:132:0: [sddz] 4096-byte physical blocks [ 43.025477] sd 0:0:25:0: [sdy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.025504] sd 0:0:57:0: [sdbe] Write Protect is off [ 43.025506] sd 0:0:57:0: [sdbe] Mode Sense: db 00 10 08 [ 43.025597] sd 0:0:82:0: [sdcc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025598] sd 0:0:82:0: [sdcc] 4096-byte physical blocks [ 43.025617] sd 0:0:133:0: [sdea] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025618] sd 0:0:133:0: [sdea] 4096-byte physical blocks [ 43.025756] sd 0:0:130:0: [sddx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025758] sd 0:0:130:0: [sddx] 4096-byte physical blocks [ 43.025767] sd 0:0:134:0: [sdeb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025768] sd 0:0:134:0: [sdeb] 4096-byte physical blocks [ 43.025802] sd 0:0:23:0: [sdw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.025859] sd 0:0:46:0: [sdat] Write Protect is off [ 43.025861] sd 0:0:46:0: [sdat] Mode Sense: db 00 10 08 [ 43.025946] sd 0:0:136:0: [sded] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.025947] sd 0:0:136:0: [sded] 4096-byte physical blocks [ 43.025954] sd 0:0:19:0: [sds] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.025956] sd 0:0:30:0: [sdad] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.025986] sd 0:0:34:0: [sdah] Write Protect is off [ 43.025997] sd 0:0:34:0: [sdah] Mode Sense: db 00 10 08 [ 43.026059] sd 0:0:33:0: [sdag] Write Protect is off [ 43.026061] sd 0:0:33:0: [sdag] Mode Sense: db 00 10 08 [ 43.026082] sd 0:0:10:0: [sdj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.026102] sd 0:0:29:0: [sdac] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.026122] sd 0:0:59:0: [sdbg] Write Protect is off [ 43.026123] sd 0:0:59:0: [sdbg] Mode Sense: db 00 10 08 [ 43.026136] sd 0:0:138:0: [sdef] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026137] sd 0:0:138:0: [sdef] 4096-byte physical blocks [ 43.026146] sd 0:0:137:0: [sdee] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026149] sd 0:0:137:0: [sdee] 4096-byte physical blocks [ 43.026168] sd 0:0:135:0: [sdec] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026170] sd 0:0:135:0: [sdec] 4096-byte physical blocks [ 43.026483] sd 0:0:125:0: [sdds] Write Protect is off [ 43.026485] sd 0:0:125:0: [sdds] Mode Sense: db 00 10 08 [ 43.026549] sd 0:0:142:0: [sdej] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026550] sd 0:0:142:0: [sdej] 4096-byte physical blocks [ 43.026573] sd 0:0:40:0: [sdan] Write Protect is off [ 43.026575] sd 0:0:40:0: [sdan] Mode Sense: db 00 10 08 [ 43.026592] sd 0:0:95:0: [sdcp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026594] sd 0:0:95:0: [sdcp] 4096-byte physical blocks [ 43.026605] sd 0:0:144:0: [sdel] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026606] sd 0:0:144:0: [sdel] 4096-byte physical blocks [ 43.026614] sd 0:0:87:0: [sdch] Write Protect is off [ 43.026616] sd 0:0:87:0: [sdch] Mode Sense: db 00 10 08 [ 43.026626] sd 0:0:92:0: [sdcm] Write Protect is off [ 43.026628] sd 0:0:92:0: [sdcm] Mode Sense: db 00 10 08 [ 43.026675] sd 0:0:141:0: [sdei] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026677] sd 0:0:141:0: [sdei] 4096-byte physical blocks [ 43.026691] sd 0:0:24:0: [sdx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026694] sd 0:0:24:0: [sdx] 4096-byte physical blocks [ 43.026715] sd 0:0:41:0: [sdao] Write Protect is off [ 43.026716] sd 0:0:41:0: [sdao] Mode Sense: db 00 10 08 [ 43.026726] sd 0:0:120:0: [sddo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026727] sd 0:0:145:0: [sdem] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026728] sd 0:0:120:0: [sddo] 4096-byte physical blocks [ 43.026729] sd 0:0:145:0: [sdem] 4096-byte physical blocks [ 43.026742] sd 0:0:74:0: [sdbu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026743] sd 0:0:74:0: [sdbu] 4096-byte physical blocks [ 43.026871] sd 0:0:147:0: [sdeo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.026872] sd 0:0:147:0: [sdeo] 4096-byte physical blocks [ 43.026882] sd 0:0:57:0: [sdbe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.026962] sd 0:0:71:0: [sdbr] Write Protect is off [ 43.026963] sd 0:0:71:0: [sdbr] Mode Sense: db 00 10 08 [ 43.027016] sd 0:0:50:0: [sdax] Write Protect is off [ 43.027017] sd 0:0:50:0: [sdax] Mode Sense: db 00 10 08 [ 43.027043] sd 0:0:91:0: [sdcl] Write Protect is off [ 43.027044] sd 0:0:91:0: [sdcl] Mode Sense: db 00 10 08 [ 43.027063] sd 0:0:93:0: [sdcn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027064] sd 0:0:93:0: [sdcn] 4096-byte physical blocks [ 43.027079] sd 0:0:146:0: [sden] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027080] sd 0:0:146:0: [sden] 4096-byte physical blocks [ 43.027087] sd 0:0:53:0: [sdba] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027088] sd 0:0:53:0: [sdba] 4096-byte physical blocks [ 43.027099] sd 0:0:76:0: [sdbw] Write Protect is off [ 43.027100] sd 0:0:76:0: [sdbw] Mode Sense: db 00 10 08 [ 43.027109] sd 0:0:68:0: [sdbo] Write Protect is off [ 43.027111] sd 0:0:68:0: [sdbo] Mode Sense: db 00 10 08 [ 43.027184] sd 0:0:79:0: [sdbz] Write Protect is off [ 43.027185] sd 0:0:79:0: [sdbz] Mode Sense: db 00 10 08 [ 43.027212] sd 0:0:36:0: [sdaj] Write Protect is off [ 43.027213] sd 0:0:36:0: [sdaj] Mode Sense: db 00 10 08 [ 43.027225] sd 0:0:66:0: [sdbm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027227] sd 0:0:66:0: [sdbm] 4096-byte physical blocks [ 43.027244] sd 0:0:84:0: [sdce] Write Protect is off [ 43.027245] sd 0:0:84:0: [sdce] Mode Sense: db 00 10 08 [ 43.027259] sd 0:0:75:0: [sdbv] Write Protect is off [ 43.027260] sd 0:0:75:0: [sdbv] Mode Sense: db 00 10 08 [ 43.027284] sd 0:0:4:0: [sdd] Write Protect is off [ 43.027286] sd 0:0:4:0: [sdd] Mode Sense: db 00 10 08 [ 43.027305] sd 0:0:46:0: [sdat] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.027347] sd 0:0:106:0: [sdda] Write Protect is off [ 43.027348] sd 0:0:34:0: [sdah] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.027349] sd 0:0:106:0: [sdda] Mode Sense: db 00 10 08 [ 43.027378] sd 0:0:149:0: [sdeq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027379] sd 0:0:149:0: [sdeq] 4096-byte physical blocks [ 43.027415] sd 0:0:94:0: [sdco] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027417] sd 0:0:94:0: [sdco] 4096-byte physical blocks [ 43.027426] sd 0:0:88:0: [sdci] Write Protect is off [ 43.027427] sd 0:0:88:0: [sdci] Mode Sense: db 00 10 08 [ 43.027435] sd 0:0:151:0: [sdes] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027436] sd 0:0:151:0: [sdes] 4096-byte physical blocks [ 43.027551] sd 0:0:97:0: [sdcr] Write Protect is off [ 43.027552] sd 0:0:97:0: [sdcr] Mode Sense: db 00 10 08 [ 43.027573] sd 0:0:16:0: [sdp] Write Protect is off [ 43.027575] sd 0:0:33:0: [sdag] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.027575] sd 0:0:16:0: [sdp] Mode Sense: db 00 10 08 [ 43.027596] sd 0:0:44:0: [sdar] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027597] sd 0:0:44:0: [sdar] 4096-byte physical blocks [ 43.027612] sd 0:0:78:0: [sdby] Write Protect is off [ 43.027613] sd 0:0:78:0: [sdby] Mode Sense: db 00 10 08 [ 43.027619] sd 1:2:0:0: [sdig] 233308160 512-byte logical blocks: (119 GB/111 GiB) [ 43.027624] sd 0:0:59:0: [sdbg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.027666] sd 0:0:85:0: [sdcf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027667] sd 0:0:85:0: [sdcf] 4096-byte physical blocks [ 43.027679] sd 0:0:100:0: [sdcu] Write Protect is off [ 43.027680] sd 0:0:100:0: [sdcu] Mode Sense: db 00 10 08 [ 43.027731] sd 0:0:18:0: [sdr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027732] sd 0:0:18:0: [sdr] 4096-byte physical blocks [ 43.027745] sd 0:0:152:0: [sdet] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.027748] sd 0:0:152:0: [sdet] 4096-byte physical blocks [ 43.027768] sd 0:0:70:0: [sdbq] Write Protect is off [ 43.027770] sd 0:0:70:0: [sdbq] Mode Sense: db 00 10 08 [ 43.027935] sd 0:0:62:0: [sdbi] Write Protect is off [ 43.027936] sd 0:0:62:0: [sdbi] Mode Sense: db 00 10 08 [ 43.027960] sd 0:0:124:0: [sddr] Write Protect is off [ 43.027962] sd 0:0:124:0: [sddr] Mode Sense: db 00 10 08 [ 43.027998] sd 0:0:83:0: [sdcd] Write Protect is off [ 43.027999] sd 0:0:83:0: [sdcd] Mode Sense: db 00 10 08 [ 43.028009] sd 0:0:153:0: [sdeu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028011] sd 0:0:153:0: [sdeu] 4096-byte physical blocks [ 43.028095] sd 0:0:123:0: [sddq] Write Protect is off [ 43.028096] sd 0:0:123:0: [sddq] Mode Sense: db 00 10 08 [ 43.028171] sd 0:0:125:0: [sdds] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.028229] sd 0:0:140:0: [sdeh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028230] sd 0:0:140:0: [sdeh] 4096-byte physical blocks [ 43.028235] sd 0:0:40:0: [sdan] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.028247] sd 0:0:56:0: [sdbd] Write Protect is off [ 43.028248] sd 0:0:56:0: [sdbd] Mode Sense: db 00 10 08 [ 43.028262] sd 0:0:242:0: [sdie] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028264] sd 0:0:242:0: [sdie] 4096-byte physical blocks [ 43.028271] sd 0:0:87:0: [sdch] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.028280] sd 0:0:92:0: [sdcm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.028338] sd 0:0:104:0: [sdcy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028341] sd 0:0:104:0: [sdcy] 4096-byte physical blocks [ 43.028371] sd 0:0:111:0: [sddf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028374] sd 0:0:111:0: [sddf] 4096-byte physical blocks [ 43.028382] sd 0:0:82:0: [sdcc] Write Protect is off [ 43.028383] sd 0:0:82:0: [sdcc] Mode Sense: db 00 10 08 [ 43.028440] sd 0:0:156:0: [sdex] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028441] sd 0:0:156:0: [sdex] 4096-byte physical blocks [ 43.028445] sd 0:0:41:0: [sdao] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.028462] sd 0:0:37:0: [sdak] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028464] sd 0:0:37:0: [sdak] 4096-byte physical blocks [ 43.028484] sd 0:0:157:0: [sdey] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028487] sd 0:0:157:0: [sdey] 4096-byte physical blocks [ 43.028538] sd 0:0:155:0: [sdew] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028539] sd 0:0:155:0: [sdew] 4096-byte physical blocks [ 43.028593] sd 0:0:127:0: [sddu] Write Protect is off [ 43.028595] sd 0:0:127:0: [sddu] Mode Sense: db 00 10 08 [ 43.028619] sd 1:2:0:0: [sdig] Write Protect is off [ 43.028621] sd 1:2:0:0: [sdig] Mode Sense: 1f 00 10 08 [ 43.028699] sd 0:0:128:0: [sddv] Write Protect is off [ 43.028701] sd 0:0:128:0: [sddv] Mode Sense: db 00 10 08 [ 43.028702] sd 0:0:129:0: [sddw] Write Protect is off [ 43.028704] sd 0:0:129:0: [sddw] Mode Sense: db 00 10 08 [ 43.028811] sd 0:0:71:0: [sdbr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.028883] sd 0:0:159:0: [sdfa] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.028886] sd 0:0:159:0: [sdfa] 4096-byte physical blocks [ 43.028920] sd 0:0:50:0: [sdax] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.028957] sd 0:0:91:0: [sdcl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029006] sd 0:0:160:0: [sdfb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029007] sd 0:0:160:0: [sdfb] 4096-byte physical blocks [ 43.029045] sd 0:0:168:0: [sdfj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029047] sd 0:0:168:0: [sdfj] 4096-byte physical blocks [ 43.029081] sd 0:0:76:0: [sdbw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029090] sd 0:0:17:0: [sdq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029091] sd 0:0:17:0: [sdq] 4096-byte physical blocks [ 43.029101] sd 0:0:161:0: [sdfc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029102] sd 0:0:161:0: [sdfc] 4096-byte physical blocks [ 43.029108] sd 0:0:68:0: [sdbo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029116] sd 0:0:90:0: [sdck] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029117] sd 0:0:90:0: [sdck] 4096-byte physical blocks [ 43.029214] sd 0:0:79:0: [sdbz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029224] sd 0:0:131:0: [sddy] Write Protect is off [ 43.029225] sd 0:0:131:0: [sddy] Mode Sense: db 00 10 08 [ 43.029244] sd 0:0:58:0: [sdbf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029246] sd 0:0:58:0: [sdbf] 4096-byte physical blocks [ 43.029262] sd 0:0:36:0: [sdaj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029277] sd 0:0:52:0: [sdaz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029278] sd 0:0:52:0: [sdaz] 4096-byte physical blocks [ 43.029291] sd 0:0:126:0: [sddt] Write Protect is off [ 43.029292] sd 0:0:126:0: [sddt] Mode Sense: db 00 10 08 [ 43.029301] sd 0:0:84:0: [sdce] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029311] sd 0:0:75:0: [sdbv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029321] sd 0:0:165:0: [sdfg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029322] sd 0:0:165:0: [sdfg] 4096-byte physical blocks [ 43.029329] sd 0:0:106:0: [sdda] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029338] sd 0:0:162:0: [sdfd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029340] sd 0:0:162:0: [sdfd] 4096-byte physical blocks [ 43.029377] sd 0:0:173:0: [sdfo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029379] sd 0:0:173:0: [sdfo] 4096-byte physical blocks [ 43.029381] sd 0:0:21:0: [sdu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029382] sd 0:0:21:0: [sdu] 4096-byte physical blocks [ 43.029407] sd 0:0:166:0: [sdfh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029409] sd 0:0:166:0: [sdfh] 4096-byte physical blocks [ 43.029421] sd 0:0:171:0: [sdfm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029421] sd 0:0:139:0: [sdeg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029422] sd 0:0:171:0: [sdfm] 4096-byte physical blocks [ 43.029423] sd 0:0:139:0: [sdeg] 4096-byte physical blocks [ 43.029442] sd 0:0:88:0: [sdci] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029447] sd 0:0:107:0: [sddb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029448] sd 0:0:107:0: [sddb] 4096-byte physical blocks [ 43.029464] sd 1:2:0:0: [sdig] Write cache: disabled, read cache: disabled, supports DPO and FUA [ 43.029493] sd 0:0:28:0: [sdab] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029495] sd 0:0:28:0: [sdab] 4096-byte physical blocks [ 43.029497] sd 0:0:167:0: [sdfi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029498] sd 0:0:167:0: [sdfi] 4096-byte physical blocks [ 43.029509] sd 0:0:112:0: [sddg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029511] sd 0:0:112:0: [sddg] 4096-byte physical blocks [ 43.029522] sd 0:0:97:0: [sdcr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029554] sd 0:0:158:0: [sdez] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029555] sd 0:0:163:0: [sdfe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029557] sd 0:0:158:0: [sdez] 4096-byte physical blocks [ 43.029559] sd 0:0:163:0: [sdfe] 4096-byte physical blocks [ 43.029565] sd 0:0:78:0: [sdby] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029591] sd 0:0:174:0: [sdfp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029592] sd 0:0:174:0: [sdfp] 4096-byte physical blocks [ 43.029615] sd 0:0:133:0: [sdea] Write Protect is off [ 43.029617] sd 0:0:133:0: [sdea] Mode Sense: db 00 10 08 [ 43.029626] sd 0:0:136:0: [sded] Write Protect is off [ 43.029627] sd 0:0:136:0: [sded] Mode Sense: db 00 10 08 [ 43.029642] sd 0:0:72:0: [sdbs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029643] sd 0:0:72:0: [sdbs] 4096-byte physical blocks [ 43.029658] sd 0:0:100:0: [sdcu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029750] sd 0:0:64:0: [sdbk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029751] sd 0:0:64:0: [sdbk] 4096-byte physical blocks [ 43.029754] sd 0:0:26:0: [sdz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029755] sd 0:0:26:0: [sdz] 4096-byte physical blocks [ 43.029760] sd 0:0:143:0: [sdek] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029761] sd 0:0:143:0: [sdek] 4096-byte physical blocks [ 43.029781] sd 0:0:170:0: [sdfl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029782] sd 0:0:170:0: [sdfl] 4096-byte physical blocks [ 43.029803] sd 0:0:132:0: [sddz] Write Protect is off [ 43.029804] sd 0:0:132:0: [sddz] Mode Sense: db 00 10 08 [ 43.029819] sd 0:0:4:0: [sdd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029819] sd 0:0:70:0: [sdbq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029836] sd 0:0:150:0: [sder] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029837] sd 0:0:150:0: [sder] 4096-byte physical blocks [ 43.029869] sd 0:0:16:0: [sdp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.029981] sd 0:0:54:0: [sdbb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.029983] sd 0:0:54:0: [sdbb] 4096-byte physical blocks [ 43.030010] sd 0:0:137:0: [sdee] Write Protect is off [ 43.030012] sd 0:0:137:0: [sdee] Mode Sense: db 00 10 08 [ 43.030017] sd 0:0:62:0: [sdbi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.030018] sd 0:0:130:0: [sddx] Write Protect is off [ 43.030019] sd 0:0:130:0: [sddx] Mode Sense: db 00 10 08 [ 43.030054] sd 0:0:32:0: [sdaf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030055] sd 0:0:63:0: [sdbj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030056] sd 0:0:32:0: [sdaf] 4096-byte physical blocks [ 43.030056] sd 0:0:63:0: [sdbj] 4096-byte physical blocks [ 43.030067] sd 0:0:172:0: [sdfn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030069] sd 0:0:172:0: [sdfn] 4096-byte physical blocks [ 43.030195] sd 0:0:148:0: [sdep] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030197] sd 0:0:148:0: [sdep] 4096-byte physical blocks [ 43.030219] sd 0:0:89:0: [sdcj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030220] sd 0:0:89:0: [sdcj] 4096-byte physical blocks [ 43.030242] sd 0:0:65:0: [sdbl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030243] sd 0:0:65:0: [sdbl] 4096-byte physical blocks [ 43.030265] sd 0:0:83:0: [sdcd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.030294] sd 0:0:95:0: [sdcp] Write Protect is off [ 43.030299] sd 0:0:95:0: [sdcp] Mode Sense: db 00 10 08 [ 43.030309] sd 0:0:67:0: [sdbn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030310] sd 0:0:67:0: [sdbn] 4096-byte physical blocks [ 43.030426] sd 0:0:175:0: [sdfq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030427] sd 0:0:175:0: [sdfq] 4096-byte physical blocks [ 43.030448] sd 0:0:124:0: [sddr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.030458] sd 0:0:154:0: [sdev] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030459] sd 0:0:154:0: [sdev] 4096-byte physical blocks [ 43.030476] sd 0:0:56:0: [sdbd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.030520] sd 0:0:74:0: [sdbu] Write Protect is off [ 43.030521] sd 0:0:74:0: [sdbu] Mode Sense: db 00 10 08 [ 43.030574] sd 0:0:82:0: [sdcc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.030705] sd 0:0:120:0: [sddo] Write Protect is off [ 43.030705] sd 0:0:135:0: [sdec] Write Protect is off [ 43.030707] sd 0:0:120:0: [sddo] Mode Sense: db 00 10 08 [ 43.030707] sd 0:0:135:0: [sdec] Mode Sense: db 00 10 08 [ 43.030788] sd 0:0:134:0: [sdeb] Write Protect is off [ 43.030790] sd 0:0:134:0: [sdeb] Mode Sense: db 00 10 08 [ 43.030838] sd 0:0:144:0: [sdel] Write Protect is off [ 43.030841] sd 0:0:144:0: [sdel] Mode Sense: db 00 10 08 [ 43.030907] sd 0:0:127:0: [sddu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.030980] sd 0:0:60:0: [sdbh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.030981] sd 0:0:60:0: [sdbh] 4096-byte physical blocks [ 43.031006] sd 0:0:123:0: [sddq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.031056] sd 0:0:80:0: [sdca] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031057] sd 0:0:80:0: [sdca] 4096-byte physical blocks [ 43.031094] sd 0:0:128:0: [sddv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.031100] sd 0:0:129:0: [sddw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.031206] sd 0:0:93:0: [sdcn] Write Protect is off [ 43.031209] sd 0:0:93:0: [sdcn] Mode Sense: db 00 10 08 [ 43.031224] sd 0:0:181:0: [sdfw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031226] sd 0:0:181:0: [sdfw] 4096-byte physical blocks [ 43.031263] sd 0:0:86:0: [sdcg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031265] sd 0:0:86:0: [sdcg] 4096-byte physical blocks [ 43.031323] sd 0:0:141:0: [sdei] Write Protect is off [ 43.031325] sd 0:0:141:0: [sdei] Mode Sense: db 00 10 08 [ 43.031413] sd 0:0:103:0: [sdcx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031414] sd 0:0:103:0: [sdcx] 4096-byte physical blocks [ 43.031431] sd 0:0:53:0: [sdba] Write Protect is off [ 43.031432] sd 0:0:53:0: [sdba] Mode Sense: db 00 10 08 [ 43.031465] sd 0:0:176:0: [sdfr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031466] sd 0:0:176:0: [sdfr] 4096-byte physical blocks [ 43.031519] sd 0:0:24:0: [sdx] Write Protect is off [ 43.031521] sd 0:0:24:0: [sdx] Mode Sense: db 00 10 08 [ 43.031545] sd 0:0:179:0: [sdfu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031547] sd 0:0:180:0: [sdfv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031548] sd 0:0:179:0: [sdfu] 4096-byte physical blocks [ 43.031548] sd 0:0:180:0: [sdfv] 4096-byte physical blocks [ 43.031579] sd 0:0:66:0: [sdbm] Write Protect is off [ 43.031580] sd 0:0:66:0: [sdbm] Mode Sense: db 00 10 08 [ 43.031659] sd 0:0:81:0: [sdcb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031660] sd 0:0:81:0: [sdcb] 4096-byte physical blocks [ 43.031701] sd 0:0:131:0: [sddy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.031776] sd 0:0:105:0: [sdcz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031777] sd 0:0:105:0: [sdcz] 4096-byte physical blocks [ 43.031796] sd 0:0:145:0: [sdem] Write Protect is off [ 43.031798] sd 0:0:138:0: [sdef] Write Protect is off [ 43.031800] sd 0:0:145:0: [sdem] Mode Sense: db 00 10 08 [ 43.031801] sd 0:0:138:0: [sdef] Mode Sense: db 00 10 08 [ 43.031817] sd 0:0:147:0: [sdeo] Write Protect is off [ 43.031818] sd 0:0:147:0: [sdeo] Mode Sense: db 00 10 08 [ 43.031842] sd 0:0:187:0: [sdgb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031844] sd 0:0:187:0: [sdgb] 4096-byte physical blocks [ 43.031853] sd 0:0:47:0: [sdau] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031854] sd 0:0:47:0: [sdau] 4096-byte physical blocks [ 43.031873] sd 0:0:178:0: [sdft] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031875] sd 0:0:178:0: [sdft] 4096-byte physical blocks [ 43.031898] sd 0:0:94:0: [sdco] Write Protect is off [ 43.031900] sd 0:0:94:0: [sdco] Mode Sense: db 00 10 08 [ 43.031942] sd 0:0:126:0: [sddt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.031953] sd 0:0:42:0: [sdap] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.031954] sd 0:0:42:0: [sdap] 4096-byte physical blocks [ 43.031979] sd 0:0:142:0: [sdej] Write Protect is off [ 43.031981] sd 0:0:142:0: [sdej] Mode Sense: db 00 10 08 [ 43.032039] sd 0:0:182:0: [sdfx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032040] sd 0:0:182:0: [sdfx] 4096-byte physical blocks [ 43.032063] sd 0:0:55:0: [sdbc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032064] sd 0:0:55:0: [sdbc] 4096-byte physical blocks [ 43.032095] sd 0:0:146:0: [sden] Write Protect is off [ 43.032097] sd 0:0:146:0: [sden] Mode Sense: db 00 10 08 [ 43.032105] sd 0:0:44:0: [sdar] Write Protect is off [ 43.032106] sd 0:0:44:0: [sdar] Mode Sense: db 00 10 08 [ 43.032119] sd 0:0:184:0: [sdfy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032121] sd 0:0:184:0: [sdfy] 4096-byte physical blocks [ 43.032128] sd 0:0:133:0: [sdea] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.032149] sd 0:0:189:0: [sdgd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032150] sd 0:0:189:0: [sdgd] 4096-byte physical blocks [ 43.032165] sd 0:0:186:0: [sdga] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032167] sd 0:0:186:0: [sdga] 4096-byte physical blocks [ 43.032202] sd 0:0:73:0: [sdbt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032205] sd 0:0:73:0: [sdbt] 4096-byte physical blocks [ 43.032262] sd 0:0:192:0: [sdgg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032263] sd 0:0:192:0: [sdgg] 4096-byte physical blocks [ 43.032289] sd 0:0:193:0: [sdgh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032290] sd 0:0:193:0: [sdgh] 4096-byte physical blocks [ 43.032300] sd 0:0:190:0: [sdge] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032301] sd 0:0:190:0: [sdge] 4096-byte physical blocks [ 43.032532] sd 0:0:194:0: [sdgi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032533] sd 0:0:194:0: [sdgi] 4096-byte physical blocks [ 43.032633] sd 0:0:132:0: [sddz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.032779] sd 0:0:18:0: [sdr] Write Protect is off [ 43.032781] sd 0:0:18:0: [sdr] Mode Sense: db 00 10 08 [ 43.032794] sd 0:0:85:0: [sdcf] Write Protect is off [ 43.032796] sd 0:0:85:0: [sdcf] Mode Sense: db 00 10 08 [ 43.032808] sd 0:0:119:0: [sddn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.032810] sd 0:0:119:0: [sddn] 4096-byte physical blocks [ 43.032869] sd 0:0:130:0: [sddx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033094] sd 0:0:203:0: [sdgr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033099] sd 0:0:203:0: [sdgr] 4096-byte physical blocks [ 43.033119] sd 0:0:200:0: [sdgo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033120] sd 0:0:200:0: [sdgo] 4096-byte physical blocks [ 43.033133] sd 0:0:241:0: [sdid] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033136] sd 0:0:241:0: [sdid] 4096-byte physical blocks [ 43.033139] sd 0:0:136:0: [sded] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033159] sd 0:0:137:0: [sdee] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033200] sd 0:0:152:0: [sdet] Write Protect is off [ 43.033202] sd 0:0:152:0: [sdet] Mode Sense: db 00 10 08 [ 43.033223] sd 0:0:164:0: [sdff] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033224] sd 0:0:164:0: [sdff] 4096-byte physical blocks [ 43.033259] sd 0:0:198:0: [sdgm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033261] sd 0:0:198:0: [sdgm] 4096-byte physical blocks [ 43.033283] sd 0:0:199:0: [sdgn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033284] sd 0:0:199:0: [sdgn] 4096-byte physical blocks [ 43.033295] sd 0:0:69:0: [sdbp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033297] sd 0:0:69:0: [sdbp] 4096-byte physical blocks [ 43.033342] sd 0:0:188:0: [sdgc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033343] sd 0:0:188:0: [sdgc] 4096-byte physical blocks [ 43.033353] sd 0:0:102:0: [sdcw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033355] sd 0:0:102:0: [sdcw] 4096-byte physical blocks [ 43.033372] sd 0:0:120:0: [sddo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033382] sd 0:0:135:0: [sdec] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033402] sd 0:0:74:0: [sdbu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033457] sd 0:0:104:0: [sdcy] Write Protect is off [ 43.033460] sd 0:0:104:0: [sdcy] Mode Sense: db 00 10 08 [ 43.033471] sd 0:0:111:0: [sddf] Write Protect is off [ 43.033472] sd 0:0:111:0: [sddf] Mode Sense: db 00 10 08 [ 43.033476] sd 0:0:140:0: [sdeh] Write Protect is off [ 43.033478] sd 0:0:140:0: [sdeh] Mode Sense: db 00 10 08 [ 43.033485] sd 0:0:96:0: [sdcq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033487] sd 0:0:96:0: [sdcq] 4096-byte physical blocks [ 43.033499] sd 0:0:95:0: [sdcp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033501] sd 0:0:202:0: [sdgq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033503] sd 0:0:202:0: [sdgq] 4096-byte physical blocks [ 43.033508] sd 0:0:196:0: [sdgk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033509] sd 0:0:196:0: [sdgk] 4096-byte physical blocks [ 43.033556] sd 0:0:77:0: [sdbx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033557] sd 0:0:77:0: [sdbx] 4096-byte physical blocks [ 43.033565] sd 0:0:37:0: [sdak] Write Protect is off [ 43.033567] sd 0:0:37:0: [sdak] Mode Sense: db 00 10 08 [ 43.033573] sd 0:0:191:0: [sdgf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033574] sd 0:0:191:0: [sdgf] 4096-byte physical blocks [ 43.033589] sd 0:0:134:0: [sdeb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.033616] sd 0:0:177:0: [sdfs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033617] sd 0:0:177:0: [sdfs] 4096-byte physical blocks [ 43.033678] sd 0:0:149:0: [sdeq] Write Protect is off [ 43.033686] sd 0:0:149:0: [sdeq] Mode Sense: db 00 10 08 [ 43.033690] sd 0:0:197:0: [sdgl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033691] sd 0:0:197:0: [sdgl] 4096-byte physical blocks [ 43.033711] sd 0:0:151:0: [sdes] Write Protect is off [ 43.033713] sd 0:0:151:0: [sdes] Mode Sense: db 00 10 08 [ 43.033717] sd 0:0:195:0: [sdgj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033719] sd 0:0:195:0: [sdgj] 4096-byte physical blocks [ 43.033746] sd 0:0:169:0: [sdfk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033747] sd 0:0:169:0: [sdfk] 4096-byte physical blocks [ 43.033812] sd 0:0:242:0: [sdie] Write Protect is off [ 43.033813] sd 0:0:242:0: [sdie] Mode Sense: db 00 10 08 [ 43.033866] sd 0:0:113:0: [sddh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033868] sd 0:0:113:0: [sddh] 4096-byte physical blocks [ 43.033882] sd 0:0:185:0: [sdfz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.033884] sd 0:0:185:0: [sdfz] 4096-byte physical blocks [ 43.034107] sd 0:0:53:0: [sdba] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.034159] sd 0:0:141:0: [sdei] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.034174] sd 0:0:24:0: [sdx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.034310] sd 0:0:93:0: [sdcn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.034414] sd 0:0:201:0: [sdgp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.034416] sd 0:0:201:0: [sdgp] 4096-byte physical blocks [ 43.034471] sd 0:0:17:0: [sdq] Write Protect is off [ 43.034473] sd 0:0:17:0: [sdq] Mode Sense: db 00 10 08 [ 43.034545] sd 0:0:109:0: [sddd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.034546] sd 0:0:109:0: [sddd] 4096-byte physical blocks [ 43.034589] sd 0:0:138:0: [sdef] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.034635] sd 0:0:168:0: [sdfj] Write Protect is off [ 43.034638] sd 0:0:168:0: [sdfj] Mode Sense: db 00 10 08 [ 43.034652] sd 0:0:156:0: [sdex] Write Protect is off [ 43.034654] sd 0:0:156:0: [sdex] Mode Sense: db 00 10 08 [ 43.034688] sd 0:0:94:0: [sdco] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.034696] sd 0:0:90:0: [sdck] Write Protect is off [ 43.034698] sd 0:0:90:0: [sdck] Mode Sense: db 00 10 08 [ 43.034822] sd 0:0:206:0: [sdgu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.034823] sd 0:0:206:0: [sdgu] 4096-byte physical blocks [ 43.034911] sd 0:0:162:0: [sdfd] Write Protect is off [ 43.034913] sd 0:0:162:0: [sdfd] Mode Sense: db 00 10 08 [ 43.035129] sd 0:0:157:0: [sdey] Write Protect is off [ 43.035130] sd 0:0:157:0: [sdey] Mode Sense: db 00 10 08 [ 43.035167] sd 0:0:44:0: [sdar] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035306] sd 0:0:107:0: [sddb] Write Protect is off [ 43.035308] sd 0:0:107:0: [sddb] Mode Sense: db 00 10 08 [ 43.035361] sd 0:0:146:0: [sden] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035377] sd 0:0:52:0: [sdaz] Write Protect is off [ 43.035378] sd 0:0:52:0: [sdaz] Mode Sense: db 00 10 08 [ 43.035401] sd 0:0:207:0: [sdgv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.035403] sd 0:0:207:0: [sdgv] 4096-byte physical blocks [ 43.035414] sd 0:0:144:0: [sdel] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035437] sd 0:0:153:0: [sdeu] Write Protect is off [ 43.035438] sd 0:0:153:0: [sdeu] Mode Sense: db 00 10 08 [ 43.035443] sd 0:0:204:0: [sdgs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.035445] sd 0:0:204:0: [sdgs] 4096-byte physical blocks [ 43.035450] sd 0:0:208:0: [sdgw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.035451] sd 0:0:208:0: [sdgw] 4096-byte physical blocks [ 43.035458] sd 0:0:58:0: [sdbf] Write Protect is off [ 43.035459] sd 0:0:58:0: [sdbf] Mode Sense: db 00 10 08 [ 43.035484] sd 0:0:205:0: [sdgt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.035486] sd 0:0:205:0: [sdgt] 4096-byte physical blocks [ 43.035504] sd 0:0:66:0: [sdbm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035514] sd 0:0:155:0: [sdew] Write Protect is off [ 43.035515] sd 0:0:155:0: [sdew] Mode Sense: db 00 10 08 [ 43.035594] sd 0:0:163:0: [sdfe] Write Protect is off [ 43.035596] sd 0:0:163:0: [sdfe] Mode Sense: db 00 10 08 [ 43.035604] sd 0:0:28:0: [sdab] Write Protect is off [ 43.035606] sd 0:0:28:0: [sdab] Mode Sense: db 00 10 08 [ 43.035613] sd 0:0:99:0: [sdct] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.035614] sd 0:0:99:0: [sdct] 4096-byte physical blocks [ 43.035618] sd 0:0:26:0: [sdz] Write Protect is off [ 43.035619] sd 0:0:26:0: [sdz] Mode Sense: db 00 10 08 [ 43.035625] sd 0:0:142:0: [sdej] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035645] sd 0:0:145:0: [sdem] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035658] sd 0:0:173:0: [sdfo] Write Protect is off [ 43.035660] sd 0:0:173:0: [sdfo] Mode Sense: db 00 10 08 [ 43.035663] sd 0:0:165:0: [sdfg] Write Protect is off [ 43.035665] sd 0:0:165:0: [sdfg] Mode Sense: db 00 10 08 [ 43.035674] sd 0:0:21:0: [sdu] Write Protect is off [ 43.035676] sd 0:0:21:0: [sdu] Mode Sense: db 00 10 08 [ 43.035721] sd 0:0:72:0: [sdbs] Write Protect is off [ 43.035722] sd 0:0:72:0: [sdbs] Mode Sense: db 00 10 08 [ 43.035750] sd 0:0:18:0: [sdr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035764] sd 0:0:167:0: [sdfi] Write Protect is off [ 43.035766] sd 0:0:167:0: [sdfi] Mode Sense: db 00 10 08 [ 43.035793] sd 0:0:147:0: [sdeo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.035808] sd 0:0:166:0: [sdfh] Write Protect is off [ 43.035810] sd 0:0:166:0: [sdfh] Mode Sense: db 00 10 08 [ 43.035869] sdig: sdig1 sdig2 sdig3 [ 43.035872] sd 0:0:159:0: [sdfa] Write Protect is off [ 43.035874] sd 0:0:159:0: [sdfa] Mode Sense: db 00 10 08 [ 43.035887] sd 0:0:112:0: [sddg] Write Protect is off [ 43.035889] sd 0:0:112:0: [sddg] Mode Sense: db 00 10 08 [ 43.035962] sd 0:0:85:0: [sdcf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.036103] sd 0:0:117:0: [sddl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.036104] sd 0:0:117:0: [sddl] 4096-byte physical blocks [ 43.036178] sd 0:0:174:0: [sdfp] Write Protect is off [ 43.036181] sd 0:0:174:0: [sdfp] Mode Sense: db 00 10 08 [ 43.036183] sd 0:0:121:0: [sddp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.036186] sd 0:0:121:0: [sddp] 4096-byte physical blocks [ 43.036234] sd 0:0:64:0: [sdbk] Write Protect is off [ 43.036236] sd 0:0:64:0: [sdbk] Mode Sense: db 00 10 08 [ 43.036238] sd 0:0:171:0: [sdfm] Write Protect is off [ 43.036239] sd 0:0:171:0: [sdfm] Mode Sense: db 00 10 08 [ 43.036368] sd 0:0:172:0: [sdfn] Write Protect is off [ 43.036369] sd 0:0:172:0: [sdfn] Mode Sense: db 00 10 08 [ 43.036393] sd 0:0:32:0: [sdaf] Write Protect is off [ 43.036395] sd 0:0:32:0: [sdaf] Mode Sense: db 00 10 08 [ 43.036449] sd 0:0:152:0: [sdet] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.036486] sd 0:0:54:0: [sdbb] Write Protect is off [ 43.036487] sd 0:0:54:0: [sdbb] Mode Sense: db 00 10 08 [ 43.036498] sd 0:0:150:0: [sder] Write Protect is off [ 43.036499] sd 0:0:150:0: [sder] Mode Sense: db 00 10 08 [ 43.036532] sd 0:0:239:0: [sdib] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.036534] sd 0:0:239:0: [sdib] 4096-byte physical blocks [ 43.036599] sd 0:0:243:0: [sdif] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.036601] sd 0:0:243:0: [sdif] 4096-byte physical blocks [ 43.036695] sd 0:0:63:0: [sdbj] Write Protect is off [ 43.036697] sd 0:0:63:0: [sdbj] Mode Sense: db 00 10 08 [ 43.036707] sd 0:0:143:0: [sdek] Write Protect is off [ 43.036708] sd 0:0:143:0: [sdek] Mode Sense: db 00 10 08 [ 43.036795] sd 0:0:89:0: [sdcj] Write Protect is off [ 43.036797] sd 0:0:89:0: [sdcj] Mode Sense: db 00 10 08 [ 43.036855] sd 0:0:104:0: [sdcy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.036870] sd 0:0:213:0: [sdhb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.036873] sd 0:0:213:0: [sdhb] 4096-byte physical blocks [ 43.036893] sd 0:0:111:0: [sddf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.036914] sd 0:0:212:0: [sdha] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.036915] sd 0:0:212:0: [sdha] 4096-byte physical blocks [ 43.036924] sd 0:0:160:0: [sdfb] Write Protect is off [ 43.036926] sd 0:0:160:0: [sdfb] Mode Sense: db 00 10 08 [ 43.036936] sd 0:0:161:0: [sdfc] Write Protect is off [ 43.036938] sd 0:0:161:0: [sdfc] Mode Sense: db 00 10 08 [ 43.036998] sd 0:0:140:0: [sdeh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.037048] sd 0:0:139:0: [sdeg] Write Protect is off [ 43.037050] sd 0:0:139:0: [sdeg] Mode Sense: db 00 10 08 [ 43.037089] sd 0:0:209:0: [sdgx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037092] sd 0:0:209:0: [sdgx] 4096-byte physical blocks [ 43.037150] sd 0:0:65:0: [sdbl] Write Protect is off [ 43.037152] sd 0:0:65:0: [sdbl] Mode Sense: db 00 10 08 [ 43.037170] sd 0:0:211:0: [sdgz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037171] sd 0:0:211:0: [sdgz] 4096-byte physical blocks [ 43.037178] sd 0:0:37:0: [sdak] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.037180] sd 0:0:170:0: [sdfl] Write Protect is off [ 43.037182] sd 0:0:170:0: [sdfl] Mode Sense: db 00 10 08 [ 43.037245] sd 0:0:210:0: [sdgy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037247] sd 0:0:210:0: [sdgy] 4096-byte physical blocks [ 43.037333] sd 0:0:158:0: [sdez] Write Protect is off [ 43.037335] sd 0:0:158:0: [sdez] Mode Sense: db 00 10 08 [ 43.037449] sd 0:0:214:0: [sdhc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037450] sd 0:0:214:0: [sdhc] 4096-byte physical blocks [ 43.037540] sd 0:0:116:0: [sddk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037542] sd 0:0:116:0: [sddk] 4096-byte physical blocks [ 43.037573] sd 0:0:151:0: [sdes] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.037646] sd 0:0:242:0: [sdie] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.037657] sd 0:0:149:0: [sdeq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.037735] sd 0:0:67:0: [sdbn] Write Protect is off [ 43.037737] sd 0:0:67:0: [sdbn] Mode Sense: db 00 10 08 [ 43.037858] sd 0:0:60:0: [sdbh] Write Protect is off [ 43.037860] sd 0:0:60:0: [sdbh] Mode Sense: db 00 10 08 [ 43.037916] sd 0:0:238:0: [sdia] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037917] sd 0:0:238:0: [sdia] 4096-byte physical blocks [ 43.037951] sd 0:0:215:0: [sdhd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037952] sd 0:0:215:0: [sdhd] 4096-byte physical blocks [ 43.037965] sd 0:0:118:0: [sddm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.037967] sd 0:0:118:0: [sddm] 4096-byte physical blocks [ 43.038003] sd 0:0:80:0: [sdca] Write Protect is off [ 43.038004] sd 0:0:80:0: [sdca] Mode Sense: db 00 10 08 [ 43.038006] sd 0:0:175:0: [sdfq] Write Protect is off [ 43.038007] sd 0:0:175:0: [sdfq] Mode Sense: db 00 10 08 [ 43.038019] sd 0:0:86:0: [sdcg] Write Protect is off [ 43.038020] sd 0:0:86:0: [sdcg] Mode Sense: db 00 10 08 [ 43.038144] sd 0:0:17:0: [sdq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.038151] sd 0:0:218:0: [sdhg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038152] sd 0:0:218:0: [sdhg] 4096-byte physical blocks [ 43.038164] sd 0:0:217:0: [sdhf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038166] sd 0:0:217:0: [sdhf] 4096-byte physical blocks [ 43.038177] sd 0:0:219:0: [sdhh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038178] sd 0:0:219:0: [sdhh] 4096-byte physical blocks [ 43.038180] sd 0:0:148:0: [sdep] Write Protect is off [ 43.038181] sd 0:0:148:0: [sdep] Mode Sense: db 00 10 08 [ 43.038211] sd 0:0:216:0: [sdhe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038215] sd 0:0:216:0: [sdhe] 4096-byte physical blocks [ 43.038228] sd 0:0:220:0: [sdhi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038229] sd 0:0:220:0: [sdhi] 4096-byte physical blocks [ 43.038242] sd 0:0:221:0: [sdhj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038243] sd 0:0:221:0: [sdhj] 4096-byte physical blocks [ 43.038252] sd 0:0:240:0: [sdic] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038254] sd 0:0:240:0: [sdic] 4096-byte physical blocks [ 43.038378] sd 0:0:168:0: [sdfj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.038430] sd 0:0:156:0: [sdex] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.038445] sd 0:0:223:0: [sdhl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038446] sd 0:0:223:0: [sdhl] 4096-byte physical blocks [ 43.038454] sd 0:0:90:0: [sdck] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.038495] sd 0:0:224:0: [sdhm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038499] sd 0:0:224:0: [sdhm] 4096-byte physical blocks [ 43.038586] sd 0:0:222:0: [sdhk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038590] sd 0:0:222:0: [sdhk] 4096-byte physical blocks [ 43.038641] sd 0:0:227:0: [sdhp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038642] sd 0:0:227:0: [sdhp] 4096-byte physical blocks [ 43.038735] sd 0:0:228:0: [sdhq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038736] sd 0:0:228:0: [sdhq] 4096-byte physical blocks [ 43.038750] sd 0:0:162:0: [sdfd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.038765] sd 0:0:226:0: [sdho] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038767] sd 0:0:226:0: [sdho] 4096-byte physical blocks [ 43.038782] sd 0:0:230:0: [sdhs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038783] sd 0:0:230:0: [sdhs] 4096-byte physical blocks [ 43.038835] sd 0:0:231:0: [sdht] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038836] sd 0:0:231:0: [sdht] 4096-byte physical blocks [ 43.038888] sd 0:0:179:0: [sdfu] Write Protect is off [ 43.038890] sd 0:0:179:0: [sdfu] Mode Sense: db 00 10 08 [ 43.038902] sd 0:0:157:0: [sdey] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.038930] sd 0:0:154:0: [sdev] Write Protect is off [ 43.038931] sd 0:0:233:0: [sdhv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038932] sd 0:0:154:0: [sdev] Mode Sense: db 00 10 08 [ 43.038932] sd 0:0:233:0: [sdhv] 4096-byte physical blocks [ 43.038980] sd 0:0:235:0: [sdhx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.038982] sd 0:0:235:0: [sdhx] 4096-byte physical blocks [ 43.038984] sd 0:0:176:0: [sdfr] Write Protect is off [ 43.038986] sd 0:0:176:0: [sdfr] Mode Sense: db 00 10 08 [ 43.039008] sd 0:0:103:0: [sdcx] Write Protect is off [ 43.039010] sd 0:0:103:0: [sdcx] Mode Sense: db 00 10 08 [ 43.039049] sd 0:0:52:0: [sdaz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039116] sd 0:0:153:0: [sdeu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039137] sd 0:0:107:0: [sddb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039150] sd 0:0:181:0: [sdfw] Write Protect is off [ 43.039152] sd 0:0:181:0: [sdfw] Mode Sense: db 00 10 08 [ 43.039159] sd 0:0:225:0: [sdhn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.039160] sd 0:0:225:0: [sdhn] 4096-byte physical blocks [ 43.039169] sd 0:0:58:0: [sdbf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039206] sd 0:0:155:0: [sdew] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039240] sd 0:0:28:0: [sdab] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039253] sd 0:0:114:0: [sddi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.039255] sd 0:0:114:0: [sddi] 4096-byte physical blocks [ 43.039277] sd 0:0:163:0: [sdfe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039283] sd 0:0:229:0: [sdhr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.039284] sd 0:0:229:0: [sdhr] 4096-byte physical blocks [ 43.039302] sd 0:0:26:0: [sdz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039326] sd 0:0:110:0: [sdde] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.039327] sd 0:0:110:0: [sdde] 4096-byte physical blocks [ 43.039329] sd 0:0:173:0: [sdfo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039336] sd 0:0:21:0: [sdu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039363] sd 0:0:165:0: [sdfg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039402] sd 0:0:190:0: [sdge] Write Protect is off [ 43.039403] sd 0:0:190:0: [sdge] Mode Sense: db 00 10 08 [ 43.039434] sd 0:0:55:0: [sdbc] Write Protect is off [ 43.039435] sd 0:0:55:0: [sdbc] Mode Sense: db 00 10 08 [ 43.039446] sd 0:0:193:0: [sdgh] Write Protect is off [ 43.039448] sd 0:0:193:0: [sdgh] Mode Sense: db 00 10 08 [ 43.039458] sd 0:0:166:0: [sdfh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039461] sd 0:0:167:0: [sdfi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039467] sd 0:0:192:0: [sdgg] Write Protect is off [ 43.039469] sd 0:0:192:0: [sdgg] Mode Sense: db 00 10 08 [ 43.039488] sd 0:0:187:0: [sdgb] Write Protect is off [ 43.039490] sd 0:0:187:0: [sdgb] Mode Sense: db 00 10 08 [ 43.039499] sd 0:0:112:0: [sddg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039502] sd 0:0:236:0: [sdhy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.039504] sd 0:0:236:0: [sdhy] 4096-byte physical blocks [ 43.039592] sd 0:0:232:0: [sdhu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.039592] sd 0:0:81:0: [sdcb] Write Protect is off [ 43.039594] sd 0:0:232:0: [sdhu] 4096-byte physical blocks [ 43.039594] sd 0:0:81:0: [sdcb] Mode Sense: db 00 10 08 [ 43.039606] sd 0:0:72:0: [sdbs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039616] sd 0:0:73:0: [sdbt] Write Protect is off [ 43.039618] sd 0:0:73:0: [sdbt] Mode Sense: db 00 10 08 [ 43.039685] sd 0:0:159:0: [sdfa] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039696] sd 0:0:178:0: [sdft] Write Protect is off [ 43.039699] sd 0:0:178:0: [sdft] Mode Sense: db 00 10 08 [ 43.039714] sd 0:0:180:0: [sdfv] Write Protect is off [ 43.039716] sd 0:0:180:0: [sdfv] Mode Sense: db 00 10 08 [ 43.039733] sd 0:0:174:0: [sdfp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039738] sd 0:0:171:0: [sdfm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039773] sd 0:0:47:0: [sdau] Write Protect is off [ 43.039774] sd 0:0:47:0: [sdau] Mode Sense: db 00 10 08 [ 43.039806] sd 0:0:119:0: [sddn] Write Protect is off [ 43.039808] sd 0:0:119:0: [sddn] Mode Sense: db 00 10 08 [ 43.039814] sd 0:0:42:0: [sdap] Write Protect is off [ 43.039817] sd 0:0:42:0: [sdap] Mode Sense: db 00 10 08 [ 43.039898] sd 0:0:172:0: [sdfn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039967] sd 0:0:105:0: [sdcz] Write Protect is off [ 43.039969] sd 0:0:105:0: [sdcz] Mode Sense: db 00 10 08 [ 43.039977] sd 0:0:32:0: [sdaf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.039978] sd 0:0:64:0: [sdbk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040048] sd 0:0:54:0: [sdbb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040066] sd 0:0:150:0: [sder] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040082] sd 0:0:234:0: [sdhw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.040083] sd 0:0:234:0: [sdhw] 4096-byte physical blocks [ 43.040200] sd 0:0:194:0: [sdgi] Write Protect is off [ 43.040202] sd 0:0:194:0: [sdgi] Mode Sense: db 00 10 08 [ 43.040294] sd 0:0:182:0: [sdfx] Write Protect is off [ 43.040296] sd 0:0:182:0: [sdfx] Mode Sense: db 00 10 08 [ 43.040314] sd 0:0:184:0: [sdfy] Write Protect is off [ 43.040315] sd 0:0:184:0: [sdfy] Mode Sense: db 00 10 08 [ 43.040322] sd 0:0:189:0: [sdgd] Write Protect is off [ 43.040323] sd 0:0:189:0: [sdgd] Mode Sense: db 00 10 08 [ 43.040341] sd 0:0:143:0: [sdek] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040347] sd 0:0:203:0: [sdgr] Write Protect is off [ 43.040349] sd 0:0:203:0: [sdgr] Mode Sense: db 00 10 08 [ 43.040352] sd 0:0:89:0: [sdcj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040400] sd 0:0:115:0: [sddj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.040401] sd 0:0:115:0: [sddj] 4096-byte physical blocks [ 43.040428] sd 0:0:49:0: [sdaw] Attached SCSI disk [ 43.040437] sd 0:0:200:0: [sdgo] Write Protect is off [ 43.040438] sd 0:0:200:0: [sdgo] Mode Sense: db 00 10 08 [ 43.040509] sd 0:0:186:0: [sdga] Write Protect is off [ 43.040510] sd 0:0:186:0: [sdga] Mode Sense: db 00 10 08 [ 43.040587] sd 0:0:164:0: [sdff] Write Protect is off [ 43.040589] sd 0:0:164:0: [sdff] Mode Sense: db 00 10 08 [ 43.040602] sd 0:0:241:0: [sdid] Write Protect is off [ 43.040604] sd 0:0:241:0: [sdid] Mode Sense: db 00 10 08 [ 43.040647] sd 0:0:161:0: [sdfc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040679] sd 0:0:237:0: [sdhz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 43.040680] sd 0:0:237:0: [sdhz] 4096-byte physical blocks [ 43.040703] sd 0:0:199:0: [sdgn] Write Protect is off [ 43.040705] sd 0:0:199:0: [sdgn] Mode Sense: db 00 10 08 [ 43.040727] sd 0:0:160:0: [sdfb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040741] sd 0:0:63:0: [sdbj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040753] sd 0:0:158:0: [sdez] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040760] sd 0:0:102:0: [sdcw] Write Protect is off [ 43.040761] sd 0:0:102:0: [sdcw] Mode Sense: db 00 10 08 [ 43.040796] sd 0:0:69:0: [sdbp] Write Protect is off [ 43.040798] sd 0:0:69:0: [sdbp] Mode Sense: db 00 10 08 [ 43.040870] sd 0:0:188:0: [sdgc] Write Protect is off [ 43.040872] sd 0:0:188:0: [sdgc] Mode Sense: db 00 10 08 [ 43.040894] sd 0:0:65:0: [sdbl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.040902] sd 0:0:139:0: [sdeg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041016] sd 0:0:170:0: [sdfl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041106] sd 0:0:96:0: [sdcq] Write Protect is off [ 43.041108] sd 0:0:96:0: [sdcq] Mode Sense: db 00 10 08 [ 43.041126] sd 0:0:67:0: [sdbn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041188] sd 0:0:198:0: [sdgm] Write Protect is off [ 43.041189] sd 0:0:198:0: [sdgm] Mode Sense: db 00 10 08 [ 43.041236] sd 0:0:51:0: [sday] Attached SCSI disk [ 43.041350] sd 0:0:60:0: [sdbh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041551] sd 0:0:175:0: [sdfq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041606] sd 0:0:80:0: [sdca] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041659] sd 0:0:191:0: [sdgf] Write Protect is off [ 43.041661] sd 0:0:191:0: [sdgf] Mode Sense: db 00 10 08 [ 43.041673] sd 0:0:86:0: [sdcg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041696] sd 0:0:196:0: [sdgk] Write Protect is off [ 43.041697] sd 0:0:196:0: [sdgk] Mode Sense: db 00 10 08 [ 43.041767] sd 0:0:77:0: [sdbx] Write Protect is off [ 43.041769] sd 0:0:77:0: [sdbx] Mode Sense: db 00 10 08 [ 43.041852] sd 0:0:148:0: [sdep] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.041874] sd 0:0:197:0: [sdgl] Write Protect is off [ 43.041876] sd 0:0:197:0: [sdgl] Mode Sense: db 00 10 08 [ 43.042075] sd 0:0:177:0: [sdfs] Write Protect is off [ 43.042077] sd 0:0:177:0: [sdfs] Mode Sense: db 00 10 08 [ 43.042083] sd 0:0:113:0: [sddh] Write Protect is off [ 43.042085] sd 0:0:113:0: [sddh] Mode Sense: db 00 10 08 [ 43.042246] sd 0:0:195:0: [sdgj] Write Protect is off [ 43.042248] sd 0:0:195:0: [sdgj] Mode Sense: db 00 10 08 [ 43.042315] sd 0:0:202:0: [sdgq] Write Protect is off [ 43.042317] sd 0:0:202:0: [sdgq] Mode Sense: db 00 10 08 [ 43.042462] sd 0:0:201:0: [sdgp] Write Protect is off [ 43.042466] sd 0:0:201:0: [sdgp] Mode Sense: db 00 10 08 [ 43.042527] sd 0:0:169:0: [sdfk] Write Protect is off [ 43.042528] sd 0:0:169:0: [sdfk] Mode Sense: db 00 10 08 [ 43.042534] sd 0:0:154:0: [sdev] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.042599] sd 0:0:185:0: [sdfz] Write Protect is off [ 43.042602] sd 0:0:185:0: [sdfz] Mode Sense: db 00 10 08 [ 43.042666] sd 0:0:103:0: [sdcx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.042755] sd 0:0:176:0: [sdfr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.042766] sd 0:0:179:0: [sdfu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.042859] sd 0:0:181:0: [sdfw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.042894] sd 0:0:109:0: [sddd] Write Protect is off [ 43.042897] sd 0:0:109:0: [sddd] Mode Sense: db 00 10 08 [ 43.043286] sd 0:0:55:0: [sdbc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043300] sd 0:0:190:0: [sdge] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043337] sd 0:0:193:0: [sdgh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043372] sd 0:0:192:0: [sdgg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043412] sd 0:0:206:0: [sdgu] Write Protect is off [ 43.043413] sd 0:0:206:0: [sdgu] Mode Sense: db 00 10 08 [ 43.043447] sd 0:0:81:0: [sdcb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043471] sd 0:0:73:0: [sdbt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043560] sd 0:0:178:0: [sdft] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043596] sd 0:0:180:0: [sdfv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043639] sd 0:0:119:0: [sddn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043642] sd 0:0:47:0: [sdau] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043678] sd 0:0:204:0: [sdgs] Write Protect is off [ 43.043680] sd 0:0:204:0: [sdgs] Mode Sense: db 00 10 08 [ 43.043683] sd 0:0:187:0: [sdgb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043724] sd 0:0:42:0: [sdap] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043825] sd 0:0:105:0: [sdcz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.043840] sd 0:0:205:0: [sdgt] Write Protect is off [ 43.043841] sd 0:0:205:0: [sdgt] Mode Sense: db 00 10 08 [ 43.043852] sd 0:0:99:0: [sdct] Write Protect is off [ 43.043853] sd 0:0:99:0: [sdct] Mode Sense: db 00 10 08 [ 43.044087] sd 0:0:189:0: [sdgd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044089] sd 0:0:194:0: [sdgi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044099] sd 0:0:184:0: [sdfy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044107] sd 0:0:182:0: [sdfx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044127] sd 0:0:203:0: [sdgr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044154] sd 0:0:208:0: [sdgw] Write Protect is off [ 43.044156] sd 0:0:208:0: [sdgw] Mode Sense: db 00 10 08 [ 43.044304] sd 0:0:186:0: [sdga] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044337] sd 0:0:117:0: [sddl] Write Protect is off [ 43.044339] sd 0:0:117:0: [sddl] Mode Sense: db 00 10 08 [ 43.044391] sd 0:0:241:0: [sdid] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044534] sd 0:0:69:0: [sdbp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044560] sd 0:0:200:0: [sdgo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044600] sd 0:0:121:0: [sddp] Write Protect is off [ 43.044602] sd 0:0:121:0: [sddp] Mode Sense: db 00 10 08 [ 43.044627] sd 0:0:188:0: [sdgc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044689] sd 0:0:207:0: [sdgv] Write Protect is off [ 43.044691] sd 0:0:102:0: [sdcw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044693] sd 0:0:207:0: [sdgv] Mode Sense: db 00 10 08 [ 43.044738] sd 0:0:164:0: [sdff] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.044810] sd 1:2:0:0: [sdig] Attached SCSI disk [ 43.044859] sd 0:0:239:0: [sdib] Write Protect is off [ 43.044861] sd 0:0:239:0: [sdib] Mode Sense: db 00 10 08 [ 43.045039] sd 0:0:243:0: [sdif] Write Protect is off [ 43.045040] sd 0:0:243:0: [sdif] Mode Sense: db 00 10 08 [ 43.045123] sd 0:0:15:0: [sdo] Attached SCSI disk [ 43.045263] sd 0:0:77:0: [sdbx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045349] sd 0:0:116:0: [sddk] Write Protect is off [ 43.045351] sd 0:0:116:0: [sddk] Mode Sense: db 00 10 08 [ 43.045449] sd 0:0:113:0: [sddh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045500] sd 0:0:96:0: [sdcq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045524] sd 0:0:213:0: [sdhb] Write Protect is off [ 43.045525] sd 0:0:118:0: [sddm] Write Protect is off [ 43.045526] sd 0:0:213:0: [sdhb] Mode Sense: db 00 10 08 [ 43.045526] sd 0:0:118:0: [sddm] Mode Sense: db 00 10 08 [ 43.045646] sd 0:0:8:0: [sdh] Attached SCSI disk [ 43.045759] sd 0:0:177:0: [sdfs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045787] sd 0:0:191:0: [sdgf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045790] sd 0:0:202:0: [sdgq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045803] sd 0:0:209:0: [sdgx] Write Protect is off [ 43.045805] sd 0:0:209:0: [sdgx] Mode Sense: db 00 10 08 [ 43.045807] sd 0:0:196:0: [sdgk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045824] sd 0:0:238:0: [sdia] Write Protect is off [ 43.045826] sd 0:0:195:0: [sdgj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045828] sd 0:0:238:0: [sdia] Mode Sense: db 00 10 08 [ 43.045830] sd 0:0:210:0: [sdgy] Write Protect is off [ 43.045833] sd 0:0:210:0: [sdgy] Mode Sense: db 00 10 08 [ 43.045838] sd 0:0:198:0: [sdgm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045849] sd 0:0:199:0: [sdgn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045866] sd 0:0:201:0: [sdgp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045883] sd 0:0:197:0: [sdgl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.045901] sd 0:0:212:0: [sdha] Write Protect is off [ 43.045903] sd 0:0:212:0: [sdha] Mode Sense: db 00 10 08 [ 43.046000] sd 0:0:185:0: [sdfz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.046006] sd 0:0:221:0: [sdhj] Write Protect is off [ 43.046007] sd 0:0:221:0: [sdhj] Mode Sense: db 00 10 08 [ 43.046015] sd 0:0:211:0: [sdgz] Write Protect is off [ 43.046016] sd 0:0:211:0: [sdgz] Mode Sense: db 00 10 08 [ 43.046023] sd 0:0:240:0: [sdic] Write Protect is off [ 43.046025] sd 0:0:240:0: [sdic] Mode Sense: db 00 10 08 [ 43.046040] sd 0:0:219:0: [sdhh] Write Protect is off [ 43.046041] sd 0:0:219:0: [sdhh] Mode Sense: db 00 10 08 [ 43.046261] sd 0:0:214:0: [sdhc] Write Protect is off [ 43.046263] sd 0:0:214:0: [sdhc] Mode Sense: db 00 10 08 [ 43.046273] sd 0:0:223:0: [sdhl] Write Protect is off [ 43.046275] sd 0:0:223:0: [sdhl] Mode Sense: db 00 10 08 [ 43.046276] sd 0:0:230:0: [sdhs] Write Protect is off [ 43.046277] sd 0:0:230:0: [sdhs] Mode Sense: db 00 10 08 [ 43.046299] sd 0:0:231:0: [sdht] Write Protect is off [ 43.046301] sd 0:0:231:0: [sdht] Mode Sense: db 00 10 08 [ 43.046387] sd 0:0:215:0: [sdhd] Write Protect is off [ 43.046389] sd 0:0:215:0: [sdhd] Mode Sense: db 00 10 08 [ 43.046410] sd 0:0:233:0: [sdhv] Write Protect is off [ 43.046412] sd 0:0:233:0: [sdhv] Mode Sense: db 00 10 08 [ 43.046416] sd 0:0:224:0: [sdhm] Write Protect is off [ 43.046418] sd 0:0:224:0: [sdhm] Mode Sense: db 00 10 08 [ 43.046428] sd 0:0:217:0: [sdhf] Write Protect is off [ 43.046429] sd 0:0:217:0: [sdhf] Mode Sense: db 00 10 08 [ 43.046438] sd 0:0:220:0: [sdhi] Write Protect is off [ 43.046439] sd 0:0:220:0: [sdhi] Mode Sense: db 00 10 08 [ 43.046665] sd 0:0:12:0: [sdl] Attached SCSI disk [ 43.046669] sd 0:0:218:0: [sdhg] Write Protect is off [ 43.046670] sd 0:0:218:0: [sdhg] Mode Sense: db 00 10 08 [ 43.046998] sd 0:0:205:0: [sdgt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047021] sd 0:0:114:0: [sddi] Write Protect is off [ 43.047026] sd 0:0:114:0: [sddi] Mode Sense: db 00 10 08 [ 43.047123] sd 0:0:222:0: [sdhk] Write Protect is off [ 43.047125] sd 0:0:222:0: [sdhk] Mode Sense: db 00 10 08 [ 43.047164] sd 0:0:99:0: [sdct] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047204] sd 0:0:232:0: [sdhu] Write Protect is off [ 43.047205] sd 0:0:232:0: [sdhu] Mode Sense: db 00 10 08 [ 43.047258] sd 0:0:117:0: [sddl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047305] sd 0:0:216:0: [sdhe] Write Protect is off [ 43.047307] sd 0:0:216:0: [sdhe] Mode Sense: db 00 10 08 [ 43.047316] sd 0:0:204:0: [sdgs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047325] sd 0:0:109:0: [sddd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047352] sd 0:0:228:0: [sdhq] Write Protect is off [ 43.047353] sd 0:0:228:0: [sdhq] Mode Sense: db 00 10 08 [ 43.047384] sd 0:0:235:0: [sdhx] Write Protect is off [ 43.047386] sd 0:0:235:0: [sdhx] Mode Sense: db 00 10 08 [ 43.047489] sd 0:0:226:0: [sdho] Write Protect is off [ 43.047491] sd 0:0:226:0: [sdho] Mode Sense: db 00 10 08 [ 43.047494] sd 0:0:121:0: [sddp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047782] sd 0:0:169:0: [sdfk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047793] sd 0:0:225:0: [sdhn] Write Protect is off [ 43.047795] sd 0:0:225:0: [sdhn] Mode Sense: db 00 10 08 [ 43.047797] sd 0:0:236:0: [sdhy] Write Protect is off [ 43.047799] sd 0:0:236:0: [sdhy] Mode Sense: db 00 10 08 [ 43.047810] sd 0:0:206:0: [sdgu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047812] sd 0:0:227:0: [sdhp] Write Protect is off [ 43.047814] sd 0:0:227:0: [sdhp] Mode Sense: db 00 10 08 [ 43.047821] sd 0:0:208:0: [sdgw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047890] sd 0:0:239:0: [sdib] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.047946] sd 0:0:115:0: [sddj] Write Protect is off [ 43.047948] sd 0:0:115:0: [sddj] Mode Sense: db 00 10 08 [ 43.048051] sd 0:0:110:0: [sdde] Write Protect is off [ 43.048053] sd 0:0:110:0: [sdde] Mode Sense: db 00 10 08 [ 43.048219] sd 0:0:234:0: [sdhw] Write Protect is off [ 43.048220] sd 0:0:234:0: [sdhw] Mode Sense: db 00 10 08 [ 43.048249] sd 0:0:243:0: [sdif] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.048276] sd 0:0:207:0: [sdgv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.048758] sd 0:0:237:0: [sdhz] Write Protect is off [ 43.048761] sd 0:0:237:0: [sdhz] Mode Sense: db 00 10 08 [ 43.048828] sd 0:0:118:0: [sddm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.048848] sd 0:0:116:0: [sddk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049216] sd 0:0:229:0: [sdhr] Write Protect is off [ 43.049218] sd 0:0:229:0: [sdhr] Mode Sense: db 00 10 08 [ 43.049239] sd 0:0:211:0: [sdgz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049248] sd 0:0:238:0: [sdia] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049430] sd 0:0:240:0: [sdic] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049522] sd 0:0:221:0: [sdhj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049528] sd 0:0:230:0: [sdhs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049562] sd 0:0:213:0: [sdhb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049565] sd 0:0:231:0: [sdht] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049611] sd 0:0:233:0: [sdhv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.049750] sd 0:0:210:0: [sdgy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050084] sd 0:0:114:0: [sddi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050187] sd 0:0:214:0: [sdhc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050205] sd 0:0:1:0: [sda] Attached SCSI disk [ 43.050210] sd 0:0:232:0: [sdhu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050217] sd 0:0:218:0: [sdhg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050220] sd 0:0:13:0: [sdm] Attached SCSI disk [ 43.050230] sd 0:0:224:0: [sdhm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050244] sd 0:0:209:0: [sdgx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050254] sd 0:0:217:0: [sdhf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050263] sd 0:0:215:0: [sdhd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050272] sd 0:0:223:0: [sdhl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050274] sd 0:0:31:0: [sdae] Attached SCSI disk [ 43.050291] sd 0:0:212:0: [sdha] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050300] sd 0:0:219:0: [sdhh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050308] sd 0:0:220:0: [sdhi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050826] sd 0:0:236:0: [sdhy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.050946] sd 0:0:115:0: [sddj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051224] sd 0:0:235:0: [sdhx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051256] sd 0:0:222:0: [sdhk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051685] sd 0:0:237:0: [sdhz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051800] sd 0:0:225:0: [sdhn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051826] sd 0:0:228:0: [sdhq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051837] sd 0:0:216:0: [sdhe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051857] sd 0:0:226:0: [sdho] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051867] sd 0:0:110:0: [sdde] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.051890] sd 0:0:227:0: [sdhp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.052265] sd 0:0:234:0: [sdhw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.052569] sd 0:0:229:0: [sdhr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.052834] sd 0:0:38:0: [sdal] Attached SCSI disk [ 43.052844] sd 0:0:20:0: [sdt] Attached SCSI disk [ 43.052852] sd 0:0:39:0: [sdam] Attached SCSI disk [ 43.053336] sd 0:0:27:0: [sdaa] Attached SCSI disk [ 43.053798] sd 0:0:35:0: [sdai] Attached SCSI disk [ 43.053975] sd 0:0:108:0: [sddc] Attached SCSI disk [ 43.054140] sd 0:0:98:0: [sdcs] Attached SCSI disk [ 43.054192] sd 0:0:2:0: [sdb] Attached SCSI disk [ 43.056647] sd 0:0:45:0: [sdas] Attached SCSI disk [ 43.057117] sd 0:0:48:0: [sdav] Attached SCSI disk [ 43.057409] sd 0:0:43:0: [sdaq] Attached SCSI disk [ 43.057716] sd 0:0:6:0: [sdf] Attached SCSI disk [ 43.058133] sd 0:0:22:0: [sdv] Attached SCSI disk [ 43.058638] sd 0:0:101:0: [sdcv] Attached SCSI disk [ 43.059269] sd 0:0:11:0: [sdk] Attached SCSI disk [ 43.059563] sd 0:0:7:0: [sdg] Attached SCSI disk [ 43.059655] sd 0:0:14:0: [sdn] Attached SCSI disk [ 43.060545] sd 0:0:5:0: [sde] Attached SCSI disk [ 43.061277] sd 0:0:25:0: [sdy] Attached SCSI disk [ 43.061947] sd 0:0:57:0: [sdbe] Attached SCSI disk [ 43.062195] sd 0:0:29:0: [sdac] Attached SCSI disk [ 43.062925] sd 0:0:19:0: [sds] Attached SCSI disk [ 43.063531] sd 0:0:23:0: [sdw] Attached SCSI disk [ 43.063578] sd 0:0:10:0: [sdj] Attached SCSI disk [ 43.063581] sd 0:0:59:0: [sdbg] Attached SCSI disk [ 43.064236] sd 0:0:46:0: [sdat] Attached SCSI disk [ 43.064390] sd 0:0:30:0: [sdad] Attached SCSI disk [ 43.064565] sd 0:0:33:0: [sdag] Attached SCSI disk [ 43.064690] sd 0:0:34:0: [sdah] Attached SCSI disk [ 43.065386] sd 0:0:9:0: [sdi] Attached SCSI disk [ 43.066975] sd 0:0:75:0: [sdbv] Attached SCSI disk [ 43.066997] sd 0:0:91:0: [sdcl] Attached SCSI disk [ 43.067281] sd 0:0:41:0: [sdao] Attached SCSI disk [ 43.067696] sd 0:0:92:0: [sdcm] Attached SCSI disk [ 43.067985] sd 0:0:125:0: [sdds] Attached SCSI disk [ 43.068086] sd 0:0:50:0: [sdax] Attached SCSI disk [ 43.068254] sd 0:0:40:0: [sdan] Attached SCSI disk [ 43.068767] sd 0:0:87:0: [sdch] Attached SCSI disk [ 43.068770] sd 0:0:83:0: [sdcd] Attached SCSI disk [ 43.068796] sd 0:0:84:0: [sdce] Attached SCSI disk [ 43.068874] sd 0:0:88:0: [sdci] Attached SCSI disk [ 43.069155] sd 0:0:79:0: [sdbz] Attached SCSI disk [ 43.069703] sd 0:0:78:0: [sdby] Attached SCSI disk [ 43.069744] sd 0:0:16:0: [sdp] Attached SCSI disk [ 43.069817] sd 0:0:124:0: [sddr] Attached SCSI disk [ 43.069827] sd 0:0:62:0: [sdbi] Attached SCSI disk [ 43.069881] sd 0:0:82:0: [sdcc] Attached SCSI disk [ 43.069921] sd 0:0:4:0: [sdd] Attached SCSI disk [ 43.070071] sd 0:0:128:0: [sddv] Attached SCSI disk [ 43.070365] sd 0:0:70:0: [sdbq] Attached SCSI disk [ 43.070501] sd 0:0:106:0: [sdda] Attached SCSI disk [ 43.070514] sd 0:0:100:0: [sdcu] Attached SCSI disk [ 43.070929] sd 0:0:68:0: [sdbo] Attached SCSI disk [ 43.071016] sd 0:0:71:0: [sdbr] Attached SCSI disk [ 43.071297] sd 0:0:36:0: [sdaj] Attached SCSI disk [ 43.071331] sd 0:0:76:0: [sdbw] Attached SCSI disk [ 43.071716] sd 0:0:97:0: [sdcr] Attached SCSI disk [ 43.071877] sd 0:0:123:0: [sddq] Attached SCSI disk [ 43.071888] sd 0:0:126:0: [sddt] Attached SCSI disk [ 43.072558] sd 0:0:130:0: [sddx] Attached SCSI disk [ 43.072671] sd 0:0:132:0: [sddz] Attached SCSI disk [ 43.072700] sd 0:0:135:0: [sdec] Attached SCSI disk [ 43.072843] sd 0:0:127:0: [sddu] Attached SCSI disk [ 43.072878] sd 0:0:56:0: [sdbd] Attached SCSI disk [ 43.072931] sd 0:0:131:0: [sddy] Attached SCSI disk [ 43.073001] sd 0:0:95:0: [sdcp] Attached SCSI disk [ 43.073371] sd 0:0:141:0: [sdei] Attached SCSI disk [ 43.073477] sd 0:0:74:0: [sdbu] Attached SCSI disk [ 43.073567] sd 0:0:120:0: [sddo] Attached SCSI disk [ 43.073846] sd 0:0:129:0: [sddw] Attached SCSI disk [ 43.073877] sd 0:0:134:0: [sdeb] Attached SCSI disk [ 43.073956] sd 0:0:133:0: [sdea] Attached SCSI disk [ 43.073972] sd 0:0:136:0: [sded] Attached SCSI disk [ 43.074035] sd 0:0:137:0: [sdee] Attached SCSI disk [ 43.074125] sd 0:0:24:0: [sdx] Attached SCSI disk [ 43.074156] sd 0:0:53:0: [sdba] Attached SCSI disk [ 43.074163] sd 0:0:144:0: [sdel] Attached SCSI disk [ 43.074493] sd 0:0:145:0: [sdem] Attached SCSI disk [ 43.074563] sd 0:0:147:0: [sdeo] Attached SCSI disk [ 43.074644] sd 0:0:93:0: [sdcn] Attached SCSI disk [ 43.074718] sd 0:0:66:0: [sdbm] Attached SCSI disk [ 43.074876] sd 0:0:142:0: [sdej] Attached SCSI disk [ 43.074884] sd 0:0:94:0: [sdco] Attached SCSI disk [ 43.075109] sd 0:0:85:0: [sdcf] Attached SCSI disk [ 43.075152] sd 0:0:18:0: [sdr] Attached SCSI disk [ 43.075391] sd 0:0:138:0: [sdef] Attached SCSI disk [ 43.075494] sd 0:0:37:0: [sdak] Attached SCSI disk [ 43.075580] sd 0:0:152:0: [sdet] Attached SCSI disk [ 43.075679] sd 0:0:149:0: [sdeq] Attached SCSI disk [ 43.075724] sd 0:0:146:0: [sden] Attached SCSI disk [ 43.075819] sd 0:0:111:0: [sddf] Attached SCSI disk [ 43.075972] sd 0:0:151:0: [sdes] Attached SCSI disk [ 43.076059] sd 0:0:104:0: [sdcy] Attached SCSI disk [ 43.076111] sd 0:0:162:0: [sdfd] Attached SCSI disk [ 43.076201] sd 0:0:242:0: [sdie] Attached SCSI disk [ 43.076282] sd 0:0:155:0: [sdew] Attached SCSI disk [ 43.076400] sd 0:0:140:0: [sdeh] Attached SCSI disk [ 43.076462] sd 0:0:90:0: [sdck] Attached SCSI disk [ 43.076582] sd 0:0:173:0: [sdfo] Attached SCSI disk [ 43.076655] sd 0:0:153:0: [sdeu] Attached SCSI disk [ 43.076694] sd 0:0:44:0: [sdar] Attached SCSI disk [ 43.076710] sd 0:0:52:0: [sdaz] Attached SCSI disk [ 43.076728] sd 0:0:171:0: [sdfm] Attached SCSI disk [ 43.076758] sd 0:0:26:0: [sdz] Attached SCSI disk [ 43.076773] sd 0:0:17:0: [sdq] Attached SCSI disk [ 43.076808] sd 0:0:107:0: [sddb] Attached SCSI disk [ 43.076827] sd 0:0:28:0: [sdab] Attached SCSI disk [ 43.077181] sd 0:0:167:0: [sdfi] Attached SCSI disk [ 43.077253] sd 0:0:58:0: [sdbf] Attached SCSI disk [ 43.077283] sd 0:0:163:0: [sdfe] Attached SCSI disk [ 43.077303] sd 0:0:156:0: [sdex] Attached SCSI disk [ 43.077353] sd 0:0:157:0: [sdey] Attached SCSI disk [ 43.077527] sd 0:0:89:0: [sdcj] Attached SCSI disk [ 43.077595] sd 0:0:32:0: [sdaf] Attached SCSI disk [ 43.077676] sd 0:0:64:0: [sdbk] Attached SCSI disk [ 43.077700] sd 0:0:166:0: [sdfh] Attached SCSI disk [ 43.077703] sd 0:0:60:0: [sdbh] Attached SCSI disk [ 43.077774] sd 0:0:54:0: [sdbb] Attached SCSI disk [ 43.077824] sd 0:0:158:0: [sdez] Attached SCSI disk [ 43.077841] sd 0:0:72:0: [sdbs] Attached SCSI disk [ 43.077881] sd 0:0:160:0: [sdfb] Attached SCSI disk [ 43.077961] sd 0:0:81:0: [sdcb] Attached SCSI disk [ 43.078029] sd 0:0:161:0: [sdfc] Attached SCSI disk [ 43.078115] sd 0:0:172:0: [sdfn] Attached SCSI disk [ 43.078157] sd 0:0:170:0: [sdfl] Attached SCSI disk [ 43.078229] sd 0:0:73:0: [sdbt] Attached SCSI disk [ 43.078240] sd 0:0:203:0: [sdgr] Attached SCSI disk [ 43.078285] sd 0:0:179:0: [sdfu] Attached SCSI disk [ 43.078311] sd 0:0:150:0: [sder] Attached SCSI disk [ 43.078320] sd 0:0:174:0: [sdfp] Attached SCSI disk [ 43.078341] sd 0:0:86:0: [sdcg] Attached SCSI disk [ 43.078354] sd 0:0:103:0: [sdcx] Attached SCSI disk [ 43.078390] sd 0:0:143:0: [sdek] Attached SCSI disk [ 43.078405] sd 0:0:139:0: [sdeg] Attached SCSI disk [ 43.078422] sd 0:0:154:0: [sdev] Attached SCSI disk [ 43.078424] sd 0:0:190:0: [sdge] Attached SCSI disk [ 43.078426] sd 0:0:112:0: [sddg] Attached SCSI disk [ 43.078448] sd 0:0:193:0: [sdgh] Attached SCSI disk [ 43.078486] sd 0:0:168:0: [sdfj] Attached SCSI disk [ 43.078500] sd 0:0:65:0: [sdbl] Attached SCSI disk [ 43.078527] sd 0:0:165:0: [sdfg] Attached SCSI disk [ 43.078615] sd 0:0:175:0: [sdfq] Attached SCSI disk [ 43.078626] sd 0:0:192:0: [sdgg] Attached SCSI disk [ 43.078656] sd 0:0:63:0: [sdbj] Attached SCSI disk [ 43.078702] sd 0:0:159:0: [sdfa] Attached SCSI disk [ 43.078752] sd 0:0:80:0: [sdca] Attached SCSI disk [ 43.078798] sd 0:0:67:0: [sdbn] Attached SCSI disk [ 43.078852] sd 0:0:148:0: [sdep] Attached SCSI disk [ 43.078882] sd 0:0:176:0: [sdfr] Attached SCSI disk [ 43.078883] sd 0:0:187:0: [sdgb] Attached SCSI disk [ 43.078959] sd 0:0:194:0: [sdgi] Attached SCSI disk [ 43.079085] sd 0:0:113:0: [sddh] Attached SCSI disk [ 43.079135] sd 0:0:180:0: [sdfv] Attached SCSI disk [ 43.079232] sd 0:0:184:0: [sdfy] Attached SCSI disk [ 43.079318] sd 0:0:178:0: [sdft] Attached SCSI disk [ 43.079373] sd 0:0:55:0: [sdbc] Attached SCSI disk [ 43.079394] sd 0:0:21:0: [sdu] Attached SCSI disk [ 43.079442] sd 0:0:197:0: [sdgl] Attached SCSI disk [ 43.079531] sd 0:0:189:0: [sdgd] Attached SCSI disk [ 43.079542] sd 0:0:181:0: [sdfw] Attached SCSI disk [ 43.079555] sd 0:0:69:0: [sdbp] Attached SCSI disk [ 43.079620] sd 0:0:241:0: [sdid] Attached SCSI disk [ 43.079747] sd 0:0:185:0: [sdfz] Attached SCSI disk [ 43.079826] sd 0:0:47:0: [sdau] Attached SCSI disk [ 43.079861] sd 0:0:202:0: [sdgq] Attached SCSI disk [ 43.079890] sd 0:0:198:0: [sdgm] Attached SCSI disk [ 43.079965] sd 0:0:200:0: [sdgo] Attached SCSI disk [ 43.080041] sd 0:0:42:0: [sdap] Attached SCSI disk [ 43.080077] sd 0:0:188:0: [sdgc] Attached SCSI disk [ 43.080163] sd 0:0:186:0: [sdga] Attached SCSI disk [ 43.080307] sd 0:0:119:0: [sddn] Attached SCSI disk [ 43.080320] sd 0:0:105:0: [sdcz] Attached SCSI disk [ 43.080512] sd 0:0:230:0: [sdhs] Attached SCSI disk [ 43.080524] sd 0:0:205:0: [sdgt] Attached SCSI disk [ 43.080586] sd 0:0:102:0: [sdcw] Attached SCSI disk [ 43.080709] sd 0:0:96:0: [sdcq] Attached SCSI disk [ 43.080718] sd 0:0:196:0: [sdgk] Attached SCSI disk [ 43.080756] sd 0:0:199:0: [sdgn] Attached SCSI disk [ 43.080804] sd 0:0:182:0: [sdfx] Attached SCSI disk [ 43.080974] sd 0:0:204:0: [sdgs] Attached SCSI disk [ 43.080988] sd 0:0:213:0: [sdhb] Attached SCSI disk [ 43.081000] sd 0:0:201:0: [sdgp] Attached SCSI disk [ 43.081057] sd 0:0:169:0: [sdfk] Attached SCSI disk [ 43.081133] sd 0:0:211:0: [sdgz] Attached SCSI disk [ 43.081180] sd 0:0:217:0: [sdhf] Attached SCSI disk [ 43.081239] sd 0:0:210:0: [sdgy] Attached SCSI disk [ 43.081275] sd 0:0:206:0: [sdgu] Attached SCSI disk [ 43.081302] sd 0:0:223:0: [sdhl] Attached SCSI disk [ 43.081412] sd 0:0:214:0: [sdhc] Attached SCSI disk [ 43.081425] sd 0:0:195:0: [sdgj] Attached SCSI disk [ 43.081449] sd 0:0:207:0: [sdgv] Attached SCSI disk [ 43.081481] sd 0:0:191:0: [sdgf] Attached SCSI disk [ 43.081488] sd 0:0:233:0: [sdhv] Attached SCSI disk [ 43.081511] sd 0:0:77:0: [sdbx] Attached SCSI disk [ 43.081522] sd 0:0:220:0: [sdhi] Attached SCSI disk [ 43.081550] sd 0:0:222:0: [sdhk] Attached SCSI disk [ 43.081594] sd 0:0:243:0: [sdif] Attached SCSI disk [ 43.081776] sd 0:0:208:0: [sdgw] Attached SCSI disk [ 43.081785] sd 0:0:215:0: [sdhd] Attached SCSI disk [ 43.081798] sd 0:0:212:0: [sdha] Attached SCSI disk [ 43.081811] sd 0:0:216:0: [sdhe] Attached SCSI disk [ 43.081849] sd 0:0:225:0: [sdhn] Attached SCSI disk [ 43.081890] sd 0:0:234:0: [sdhw] Attached SCSI disk [ 43.081910] sd 0:0:121:0: [sddp] Attached SCSI disk [ 43.082008] sd 0:0:229:0: [sdhr] Attached SCSI disk [ 43.082048] sd 0:0:226:0: [sdho] Attached SCSI disk [ 43.082107] sd 0:0:219:0: [sdhh] Attached SCSI disk [ 43.082120] sd 0:0:209:0: [sdgx] Attached SCSI disk [ 43.082134] sd 0:0:228:0: [sdhq] Attached SCSI disk [ 43.082346] sd 0:0:117:0: [sddl] Attached SCSI disk [ 43.082356] sd 0:0:118:0: [sddm] Attached SCSI disk [ 43.082503] sd 0:0:164:0: [sdff] Attached SCSI disk [ 43.082520] sd 0:0:238:0: [sdia] Attached SCSI disk [ 43.082564] sd 0:0:177:0: [sdfs] Attached SCSI disk [ 43.082592] sd 0:0:109:0: [sddd] Attached SCSI disk [ 43.082672] sd 0:0:239:0: [sdib] Attached SCSI disk [ 43.082678] sd 0:0:235:0: [sdhx] Attached SCSI disk [ 43.082721] sd 0:0:240:0: [sdic] Attached SCSI disk [ 43.082839] sd 0:0:116:0: [sddk] Attached SCSI disk [ 43.082928] sd 0:0:221:0: [sdhj] Attached SCSI disk [ 43.082937] sd 0:0:231:0: [sdht] Attached SCSI disk [ 43.083000] sd 0:0:114:0: [sddi] Attached SCSI disk [ 43.083061] sd 0:0:224:0: [sdhm] Attached SCSI disk [ 43.083149] sd 0:0:218:0: [sdhg] Attached SCSI disk [ 43.083329] sd 0:0:236:0: [sdhy] Attached SCSI disk [ 43.083479] sd 0:0:227:0: [sdhp] Attached SCSI disk [ 43.083647] sd 0:0:115:0: [sddj] Attached SCSI disk [ 43.083668] sd 0:0:99:0: [sdct] Attached SCSI disk [ 43.084050] sd 0:0:232:0: [sdhu] Attached SCSI disk [ 43.084441] sd 0:0:237:0: [sdhz] Attached SCSI disk [ 43.084773] sd 0:0:110:0: [sdde] Attached SCSI disk [ 43.276514] sd 0:0:3:0: [sdc] 4096-byte physical blocks [ 43.278855] sd 0:0:3:0: [sdc] Write Protect is off [ 43.278857] sd 0:0:3:0: [sdc] Mode Sense: db 00 10 08 [ 43.280149] sd 0:0:3:0: [sdc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 43.292486] sd 0:0:3:0: [sdc] Attached SCSI disk [ 50.910673] EXT4-fs (sdig2): mounted filesystem with ordered data mode. Opts: (null) [ 51.216632] systemd-journald[354]: Received SIGTERM from PID 1 (systemd). [ 51.246505] SELinux: Disabled at runtime. [ 51.251680] SELinux: Unregistering netfilter hooks [ 51.295190] type=1404 audit(1547777752.798:2): selinux=0 auid=4294967295 ses=4294967295 [ 51.318886] ip_tables: (C) 2000-2006 Netfilter Core Team [ 51.324678] systemd[1]: Inserted module 'ip_tables' [ 51.426740] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 51.426906] EXT4-fs (sdig2): re-mounted. Opts: (null) [ 51.446420] knem 1.1.3.90mlnx1: initialized [ 51.456119] systemd-journald[14682]: Received request to flush runtime journal from PID 1 [ 51.587506] ACPI Error: No handler for Region [SYSI] (ffff927529e7ab40) [IPMI] (20130517/evregion-162) [ 51.599925] ACPI Error: Region IPMI (ID=7) has no handler (20130517/exfldio-305) [ 51.610160] ACPI Error: Method parse/execution failed [\_SB_.PMI0._GHL] (Node ffff926529e7b5a0), AE_NOT_EXIST (20130517/psparse-536) [ 51.629081] ACPI Error: Method parse/execution failed [\_SB_.PMI0._PMC] (Node ffff926529e7b500), AE_NOT_EXIST (20130517/psparse-536) [ 51.648017] ACPI Exception: AE_NOT_EXIST, Evaluating _PMC (20130517/power_meter-753) [ 51.661814] ipmi message handler version 39.2 [ 51.662710] scsi 0:0:0:0: Attached scsi generic sg0 type 13 [ 51.663010] sd 0:0:1:0: Attached scsi generic sg1 type 0 [ 51.663184] sd 0:0:2:0: Attached scsi generic sg2 type 0 [ 51.663967] sd 0:0:3:0: Attached scsi generic sg3 type 0 [ 51.664088] sd 0:0:4:0: Attached scsi generic sg4 type 0 [ 51.665533] sd 0:0:5:0: Attached scsi generic sg5 type 0 [ 51.667470] sd 0:0:6:0: Attached scsi generic sg6 type 0 [ 51.670900] sd 0:0:7:0: Attached scsi generic sg7 type 0 [ 51.672488] sd 0:0:8:0: Attached scsi generic sg8 type 0 [ 51.672778] sd 0:0:9:0: Attached scsi generic sg9 type 0 [ 51.673064] sd 0:0:10:0: Attached scsi generic sg10 type 0 [ 51.673333] sd 0:0:11:0: Attached scsi generic sg11 type 0 [ 51.673936] sd 0:0:12:0: Attached scsi generic sg12 type 0 [ 51.674262] sd 0:0:13:0: Attached scsi generic sg13 type 0 [ 51.674546] sd 0:0:14:0: Attached scsi generic sg14 type 0 [ 51.674810] sd 0:0:15:0: Attached scsi generic sg15 type 0 [ 51.675009] sd 0:0:16:0: Attached scsi generic sg16 type 0 [ 51.675314] sd 0:0:17:0: Attached scsi generic sg17 type 0 [ 51.675632] sd 0:0:18:0: Attached scsi generic sg18 type 0 [ 51.675911] sd 0:0:19:0: Attached scsi generic sg19 type 0 [ 51.676390] sd 0:0:20:0: Attached scsi generic sg20 type 0 [ 51.676819] sd 0:0:21:0: Attached scsi generic sg21 type 0 [ 51.677168] sd 0:0:22:0: Attached scsi generic sg22 type 0 [ 51.677726] sd 0:0:23:0: Attached scsi generic sg23 type 0 [ 51.678346] sd 0:0:24:0: Attached scsi generic sg24 type 0 [ 51.678873] sd 0:0:25:0: Attached scsi generic sg25 type 0 [ 51.679278] sd 0:0:26:0: Attached scsi generic sg26 type 0 [ 51.679841] sd 0:0:27:0: Attached scsi generic sg27 type 0 [ 51.680312] sd 0:0:28:0: Attached scsi generic sg28 type 0 [ 51.680713] sd 0:0:29:0: Attached scsi generic sg29 type 0 [ 51.681028] sd 0:0:30:0: Attached scsi generic sg30 type 0 [ 51.681288] sd 0:0:31:0: Attached scsi generic sg31 type 0 [ 51.682560] sd 0:0:32:0: Attached scsi generic sg32 type 0 [ 51.683023] sd 0:0:33:0: Attached scsi generic sg33 type 0 [ 51.683405] sd 0:0:34:0: Attached scsi generic sg34 type 0 [ 51.683969] sd 0:0:35:0: Attached scsi generic sg35 type 0 [ 51.684541] sd 0:0:36:0: Attached scsi generic sg36 type 0 [ 51.685055] sd 0:0:37:0: Attached scsi generic sg37 type 0 [ 51.685635] sd 0:0:38:0: Attached scsi generic sg38 type 0 [ 51.687070] sd 0:0:39:0: Attached scsi generic sg39 type 0 [ 51.687275] sd 0:0:40:0: Attached scsi generic sg40 type 0 [ 51.688668] sd 0:0:41:0: Attached scsi generic sg41 type 0 [ 51.689443] sd 0:0:42:0: Attached scsi generic sg42 type 0 [ 51.690033] sd 0:0:43:0: Attached scsi generic sg43 type 0 [ 51.690426] sd 0:0:44:0: Attached scsi generic sg44 type 0 [ 51.690880] sd 0:0:45:0: Attached scsi generic sg45 type 0 [ 51.691301] sd 0:0:46:0: Attached scsi generic sg46 type 0 [ 51.693375] sd 0:0:47:0: Attached scsi generic sg47 type 0 [ 51.693870] sd 0:0:48:0: Attached scsi generic sg48 type 0 [ 51.694089] sd 0:0:49:0: Attached scsi generic sg49 type 0 [ 51.694462] sd 0:0:50:0: Attached scsi generic sg50 type 0 [ 51.694963] sd 0:0:51:0: Attached scsi generic sg51 type 0 [ 51.695620] sd 0:0:52:0: Attached scsi generic sg52 type 0 [ 51.696469] sd 0:0:53:0: Attached scsi generic sg53 type 0 [ 51.696853] sd 0:0:54:0: Attached scsi generic sg54 type 0 [ 51.697065] sd 0:0:55:0: Attached scsi generic sg55 type 0 [ 51.697306] sd 0:0:56:0: Attached scsi generic sg56 type 0 [ 51.698410] sd 0:0:57:0: Attached scsi generic sg57 type 0 [ 51.698610] sd 0:0:58:0: Attached scsi generic sg58 type 0 [ 51.701444] sd 0:0:59:0: Attached scsi generic sg59 type 0 [ 51.703227] sd 0:0:60:0: Attached scsi generic sg60 type 0 [ 51.703564] scsi 0:0:61:0: Attached scsi generic sg61 type 13 [ 51.703941] sd 0:0:62:0: Attached scsi generic sg62 type 0 [ 51.705284] sd 0:0:63:0: Attached scsi generic sg63 type 0 [ 51.705992] sd 0:0:64:0: Attached scsi generic sg64 type 0 [ 51.706634] sd 0:0:65:0: Attached scsi generic sg65 type 0 [ 51.706885] sd 0:0:66:0: Attached scsi generic sg66 type 0 [ 51.709103] sd 0:0:67:0: Attached scsi generic sg67 type 0 [ 51.711520] sd 0:0:68:0: Attached scsi generic sg68 type 0 [ 51.714444] sd 0:0:69:0: Attached scsi generic sg69 type 0 [ 51.714791] sd 0:0:70:0: Attached scsi generic sg70 type 0 [ 51.715243] sd 0:0:71:0: Attached scsi generic sg71 type 0 [ 51.718487] sd 0:0:72:0: Attached scsi generic sg72 type 0 [ 51.718799] sd 0:0:73:0: Attached scsi generic sg73 type 0 [ 51.719342] sd 0:0:74:0: Attached scsi generic sg74 type 0 [ 51.719925] sd 0:0:75:0: Attached scsi generic sg75 type 0 [ 51.720361] sd 0:0:76:0: Attached scsi generic sg76 type 0 [ 51.720600] sd 0:0:77:0: Attached scsi generic sg77 type 0 [ 51.720841] sd 0:0:78:0: Attached scsi generic sg78 type 0 [ 51.720976] sd 0:0:79:0: Attached scsi generic sg79 type 0 [ 51.721024] sd 0:0:80:0: Attached scsi generic sg80 type 0 [ 51.721082] sd 0:0:81:0: Attached scsi generic sg81 type 0 [ 51.721141] sd 0:0:82:0: Attached scsi generic sg82 type 0 [ 51.721268] sd 0:0:83:0: Attached scsi generic sg83 type 0 [ 51.721450] sd 0:0:84:0: Attached scsi generic sg84 type 0 [ 51.721779] sd 0:0:85:0: Attached scsi generic sg85 type 0 [ 51.722233] sd 0:0:86:0: Attached scsi generic sg86 type 0 [ 51.722558] sd 0:0:87:0: Attached scsi generic sg87 type 0 [ 51.722802] sd 0:0:88:0: Attached scsi generic sg88 type 0 [ 51.723024] sd 0:0:89:0: Attached scsi generic sg89 type 0 [ 51.723231] sd 0:0:90:0: Attached scsi generic sg90 type 0 [ 51.725223] sd 0:0:91:0: Attached scsi generic sg91 type 0 [ 51.725692] sd 0:0:92:0: Attached scsi generic sg92 type 0 [ 51.726255] sd 0:0:93:0: Attached scsi generic sg93 type 0 [ 51.727691] sd 0:0:94:0: Attached scsi generic sg94 type 0 [ 51.728072] sd 0:0:95:0: Attached scsi generic sg95 type 0 [ 51.728532] sd 0:0:96:0: Attached scsi generic sg96 type 0 [ 51.728831] sd 0:0:97:0: Attached scsi generic sg97 type 0 [ 51.729028] sd 0:0:98:0: Attached scsi generic sg98 type 0 [ 51.729224] sd 0:0:99:0: Attached scsi generic sg99 type 0 [ 51.729351] sd 0:0:100:0: Attached scsi generic sg100 type 0 [ 51.729446] sd 0:0:101:0: Attached scsi generic sg101 type 0 [ 51.729518] sd 0:0:102:0: Attached scsi generic sg102 type 0 [ 51.729622] sd 0:0:103:0: Attached scsi generic sg103 type 0 [ 51.729668] sd 0:0:104:0: Attached scsi generic sg104 type 0 [ 51.729720] sd 0:0:105:0: Attached scsi generic sg105 type 0 [ 51.729770] sd 0:0:106:0: Attached scsi generic sg106 type 0 [ 51.729856] sd 0:0:107:0: Attached scsi generic sg107 type 0 [ 51.729901] sd 0:0:108:0: Attached scsi generic sg108 type 0 [ 51.729947] sd 0:0:109:0: Attached scsi generic sg109 type 0 [ 51.729993] sd 0:0:110:0: Attached scsi generic sg110 type 0 [ 51.730035] sd 0:0:111:0: Attached scsi generic sg111 type 0 [ 51.730074] sd 0:0:112:0: Attached scsi generic sg112 type 0 [ 51.730113] sd 0:0:113:0: Attached scsi generic sg113 type 0 [ 51.730149] sd 0:0:114:0: Attached scsi generic sg114 type 0 [ 51.730186] sd 0:0:115:0: Attached scsi generic sg115 type 0 [ 51.730227] sd 0:0:116:0: Attached scsi generic sg116 type 0 [ 51.730264] sd 0:0:117:0: Attached scsi generic sg117 type 0 [ 51.730302] sd 0:0:118:0: Attached scsi generic sg118 type 0 [ 51.730339] sd 0:0:119:0: Attached scsi generic sg119 type 0 [ 51.730377] sd 0:0:120:0: Attached scsi generic sg120 type 0 [ 51.730424] sd 0:0:121:0: Attached scsi generic sg121 type 0 [ 51.730464] scsi 0:0:122:0: Attached scsi generic sg122 type 13 [ 51.730505] sd 0:0:123:0: Attached scsi generic sg123 type 0 [ 51.730545] sd 0:0:124:0: Attached scsi generic sg124 type 0 [ 51.730587] sd 0:0:125:0: Attached scsi generic sg125 type 0 [ 51.730625] sd 0:0:126:0: Attached scsi generic sg126 type 0 [ 51.730668] sd 0:0:127:0: Attached scsi generic sg127 type 0 [ 51.730709] sd 0:0:128:0: Attached scsi generic sg128 type 0 [ 51.730750] sd 0:0:129:0: Attached scsi generic sg129 type 0 [ 51.730791] sd 0:0:130:0: Attached scsi generic sg130 type 0 [ 51.730831] sd 0:0:131:0: Attached scsi generic sg131 type 0 [ 51.730870] sd 0:0:132:0: Attached scsi generic sg132 type 0 [ 51.730910] sd 0:0:133:0: Attached scsi generic sg133 type 0 [ 51.730950] sd 0:0:134:0: Attached scsi generic sg134 type 0 [ 51.730989] sd 0:0:135:0: Attached scsi generic sg135 type 0 [ 51.731028] sd 0:0:136:0: Attached scsi generic sg136 type 0 [ 51.731069] sd 0:0:137:0: Attached scsi generic sg137 type 0 [ 51.731111] sd 0:0:138:0: Attached scsi generic sg138 type 0 [ 51.731151] sd 0:0:139:0: Attached scsi generic sg139 type 0 [ 51.731194] sd 0:0:140:0: Attached scsi generic sg140 type 0 [ 51.731233] sd 0:0:141:0: Attached scsi generic sg141 type 0 [ 51.731270] sd 0:0:142:0: Attached scsi generic sg142 type 0 [ 51.731306] sd 0:0:143:0: Attached scsi generic sg143 type 0 [ 51.731342] sd 0:0:144:0: Attached scsi generic sg144 type 0 [ 51.731378] sd 0:0:145:0: Attached scsi generic sg145 type 0 [ 51.731413] sd 0:0:146:0: Attached scsi generic sg146 type 0 [ 51.731448] sd 0:0:147:0: Attached scsi generic sg147 type 0 [ 51.731486] sd 0:0:148:0: Attached scsi generic sg148 type 0 [ 51.731521] sd 0:0:149:0: Attached scsi generic sg149 type 0 [ 51.731556] sd 0:0:150:0: Attached scsi generic sg150 type 0 [ 51.731593] sd 0:0:151:0: Attached scsi generic sg151 type 0 [ 51.731630] sd 0:0:152:0: Attached scsi generic sg152 type 0 [ 51.731665] sd 0:0:153:0: Attached scsi generic sg153 type 0 [ 51.731699] sd 0:0:154:0: Attached scsi generic sg154 type 0 [ 51.731736] sd 0:0:155:0: Attached scsi generic sg155 type 0 [ 51.731770] sd 0:0:156:0: Attached scsi generic sg156 type 0 [ 51.731805] sd 0:0:157:0: Attached scsi generic sg157 type 0 [ 51.731840] sd 0:0:158:0: Attached scsi generic sg158 type 0 [ 51.731876] sd 0:0:159:0: Attached scsi generic sg159 type 0 [ 51.731913] sd 0:0:160:0: Attached scsi generic sg160 type 0 [ 51.731947] sd 0:0:161:0: Attached scsi generic sg161 type 0 [ 51.731983] sd 0:0:162:0: Attached scsi generic sg162 type 0 [ 51.732020] sd 0:0:163:0: Attached scsi generic sg163 type 0 [ 51.732055] sd 0:0:164:0: Attached scsi generic sg164 type 0 [ 51.732090] sd 0:0:165:0: Attached scsi generic sg165 type 0 [ 51.732126] sd 0:0:166:0: Attached scsi generic sg166 type 0 [ 51.732160] sd 0:0:167:0: Attached scsi generic sg167 type 0 [ 51.732201] sd 0:0:168:0: Attached scsi generic sg168 type 0 [ 51.732245] sd 0:0:169:0: Attached scsi generic sg169 type 0 [ 51.732284] sd 0:0:170:0: Attached scsi generic sg170 type 0 [ 51.732321] sd 0:0:171:0: Attached scsi generic sg171 type 0 [ 51.732357] sd 0:0:172:0: Attached scsi generic sg172 type 0 [ 51.732562] sd 0:0:173:0: Attached scsi generic sg173 type 0 [ 51.732602] sd 0:0:174:0: Attached scsi generic sg174 type 0 [ 51.732642] sd 0:0:175:0: Attached scsi generic sg175 type 0 [ 51.732683] sd 0:0:176:0: Attached scsi generic sg176 type 0 [ 51.732725] sd 0:0:177:0: Attached scsi generic sg177 type 0 [ 51.732766] sd 0:0:178:0: Attached scsi generic sg178 type 0 [ 51.732806] sd 0:0:179:0: Attached scsi generic sg179 type 0 [ 51.732846] sd 0:0:180:0: Attached scsi generic sg180 type 0 [ 51.732888] sd 0:0:181:0: Attached scsi generic sg181 type 0 [ 51.732927] sd 0:0:182:0: Attached scsi generic sg182 type 0 [ 51.732967] scsi 0:0:183:0: Attached scsi generic sg183 type 13 [ 51.733008] sd 0:0:184:0: Attached scsi generic sg184 type 0 [ 51.733092] sd 0:0:185:0: Attached scsi generic sg185 type 0 [ 51.733224] sd 0:0:186:0: Attached scsi generic sg186 type 0 [ 51.733472] sd 0:0:187:0: Attached scsi generic sg187 type 0 [ 51.733695] sd 0:0:188:0: Attached scsi generic sg188 type 0 [ 51.734021] sd 0:0:189:0: Attached scsi generic sg189 type 0 [ 51.734432] sd 0:0:190:0: Attached scsi generic sg190 type 0 [ 51.735244] sd 0:0:191:0: Attached scsi generic sg191 type 0 [ 51.735769] sd 0:0:192:0: Attached scsi generic sg192 type 0 [ 51.736367] sd 0:0:193:0: Attached scsi generic sg193 type 0 [ 51.737240] sd 0:0:194:0: Attached scsi generic sg194 type 0 [ 51.737724] sd 0:0:195:0: Attached scsi generic sg195 type 0 [ 51.738243] sd 0:0:196:0: Attached scsi generic sg196 type 0 [ 51.738940] sd 0:0:197:0: Attached scsi generic sg197 type 0 [ 51.739384] sd 0:0:198:0: Attached scsi generic sg198 type 0 [ 51.740359] sd 0:0:199:0: Attached scsi generic sg199 type 0 [ 51.740916] sd 0:0:200:0: Attached scsi generic sg200 type 0 [ 51.741426] sd 0:0:201:0: Attached scsi generic sg201 type 0 [ 51.741956] sd 0:0:202:0: Attached scsi generic sg202 type 0 [ 51.742561] sd 0:0:203:0: Attached scsi generic sg203 type 0 [ 51.742998] sd 0:0:204:0: Attached scsi generic sg204 type 0 [ 51.743463] sd 0:0:205:0: Attached scsi generic sg205 type 0 [ 51.743890] sd 0:0:206:0: Attached scsi generic sg206 type 0 [ 51.744433] sd 0:0:207:0: Attached scsi generic sg207 type 0 [ 51.745179] sd 0:0:208:0: Attached scsi generic sg208 type 0 [ 51.746031] sd 0:0:209:0: Attached scsi generic sg209 type 0 [ 51.746556] sd 0:0:210:0: Attached scsi generic sg210 type 0 [ 51.747069] sd 0:0:211:0: Attached scsi generic sg211 type 0 [ 51.747672] sd 0:0:212:0: Attached scsi generic sg212 type 0 [ 51.748220] sd 0:0:213:0: Attached scsi generic sg213 type 0 [ 51.748851] sd 0:0:214:0: Attached scsi generic sg214 type 0 [ 51.749313] sd 0:0:215:0: Attached scsi generic sg215 type 0 [ 51.749949] sd 0:0:216:0: Attached scsi generic sg216 type 0 [ 51.750364] sd 0:0:217:0: Attached scsi generic sg217 type 0 [ 51.751479] sd 0:0:218:0: Attached scsi generic sg218 type 0 [ 51.751894] sd 0:0:219:0: Attached scsi generic sg219 type 0 [ 51.752293] sd 0:0:220:0: Attached scsi generic sg220 type 0 [ 51.756461] sd 0:0:221:0: Attached scsi generic sg221 type 0 [ 51.756698] sd 0:0:222:0: Attached scsi generic sg222 type 0 [ 51.757057] sd 0:0:223:0: Attached scsi generic sg223 type 0 [ 51.758981] sd 0:0:224:0: Attached scsi generic sg224 type 0 [ 51.759493] sd 0:0:225:0: Attached scsi generic sg225 type 0 [ 51.767243] sd 0:0:226:0: Attached scsi generic sg226 type 0 [ 51.768751] sd 0:0:227:0: Attached scsi generic sg227 type 0 [ 51.769291] sd 0:0:228:0: Attached scsi generic sg228 type 0 [ 51.770604] sd 0:0:229:0: Attached scsi generic sg229 type 0 [ 51.771046] sd 0:0:230:0: Attached scsi generic sg230 type 0 [ 51.772532] sd 0:0:231:0: Attached scsi generic sg231 type 0 [ 51.772925] sd 0:0:232:0: Attached scsi generic sg232 type 0 [ 51.773280] sd 0:0:233:0: Attached scsi generic sg233 type 0 [ 51.775162] sd 0:0:234:0: Attached scsi generic sg234 type 0 [ 51.775684] sd 0:0:235:0: Attached scsi generic sg235 type 0 [ 51.777023] sd 0:0:236:0: Attached scsi generic sg236 type 0 [ 51.778784] sd 0:0:237:0: Attached scsi generic sg237 type 0 [ 51.779218] sd 0:0:238:0: Attached scsi generic sg238 type 0 [ 51.780203] sd 0:0:239:0: Attached scsi generic sg239 type 0 [ 51.780692] sd 0:0:240:0: Attached scsi generic sg240 type 0 [ 51.781253] sd 0:0:241:0: Attached scsi generic sg241 type 0 [ 51.781881] sd 0:0:242:0: Attached scsi generic sg242 type 0 [ 51.782976] sd 0:0:243:0: Attached scsi generic sg243 type 0 [ 51.783612] scsi 0:0:244:0: Attached scsi generic sg244 type 13 [ 51.784060] sd 1:2:0:0: Attached scsi generic sg245 type 0 [ 51.837495] device-mapper: uevent: version 1.0.3 [ 51.838319] device-mapper: ioctl: 4.37.1-ioctl (2018-04-03) initialised: dm-devel@redhat.com [ 51.955604] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977647] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977697] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977743] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977789] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977836] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977883] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977929] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.977975] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.978021] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.978068] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.978116] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.978163] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.978222] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.978275] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979232] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979289] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979480] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979529] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979567] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979604] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979641] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979678] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979716] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979754] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979791] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979828] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979866] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979904] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979941] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.979979] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980017] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980059] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980095] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980132] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980170] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980206] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980253] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980296] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980336] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980377] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980415] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980454] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980505] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980547] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980588] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980628] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980670] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980716] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980757] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980798] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980838] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980881] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.980926] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982102] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982145] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982189] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982236] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982278] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982319] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982362] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982403] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982445] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982488] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982529] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982568] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982608] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982648] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982692] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982734] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982777] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982830] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982872] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982916] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982956] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.982997] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983039] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983080] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983124] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983164] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983204] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983250] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983293] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983337] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983379] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983418] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983459] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983500] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983542] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983582] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983623] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983664] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983707] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983748] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983788] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983828] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983868] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983908] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983949] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.983991] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984033] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984074] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984117] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984158] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984208] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984253] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984295] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984339] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984381] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984467] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984507] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984547] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984588] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984627] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984667] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984706] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984746] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984788] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984827] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984868] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984910] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984952] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.984991] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985031] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985070] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985108] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985166] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985206] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985247] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985287] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985326] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985366] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985405] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985445] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985485] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.985526] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986552] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986640] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986707] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986758] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986807] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986858] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986908] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.986961] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987012] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987062] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987111] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987163] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987220] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987270] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987322] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987373] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987432] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987485] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987560] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987619] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987675] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987731] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987786] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987843] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987902] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.987958] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988007] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988049] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988091] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988135] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988178] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988260] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988310] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988362] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988407] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988451] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988495] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988544] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988590] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988634] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988676] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988719] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988762] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988803] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988843] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988887] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988930] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.988970] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989014] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989055] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989092] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989137] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989177] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989224] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989263] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989441] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989486] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989531] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989573] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989618] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989662] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989705] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989747] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989791] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989836] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989883] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989920] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.989959] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990003] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990047] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990090] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990134] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990176] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990225] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990273] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990317] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990359] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990402] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990440] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990483] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990532] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990577] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990632] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990675] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990719] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990763] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990804] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990842] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990880] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990924] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.990963] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.991000] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.991038] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.991074] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.991113] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.991151] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.991189] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.991230] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.996832] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.996916] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.996966] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997020] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997063] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997104] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997145] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997188] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997263] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997318] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997369] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997414] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997461] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997507] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997565] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997607] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997657] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997705] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997753] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997796] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997842] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997884] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997927] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.997978] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998020] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998058] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998095] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998133] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998178] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998260] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998305] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998370] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998624] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998668] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998713] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998761] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998800] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998843] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998897] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998942] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.998989] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999030] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999069] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999109] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999147] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999191] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999241] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999282] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999319] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999358] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999395] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999433] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999472] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999510] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999549] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999593] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999637] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999680] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999721] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999765] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999820] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999865] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999912] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 51.999956] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.000000] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.000045] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.000088] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.320158] ipmi device interface [ 52.373310] IPMI System Interface driver. [ 52.373331] ipmi_si ipmi_si.0: ipmi_platform: probing via SMBIOS [ 52.373333] ipmi_si: SMBIOS: io 0xca8 regsize 1 spacing 4 irq 10 [ 52.373334] ipmi_si: Adding SMBIOS-specified kcs state machine [ 52.373364] ipmi_si IPI0001:00: ipmi_platform: probing via ACPI [ 52.373391] ipmi_si IPI0001:00: [io 0x0ca8] regsize 1 spacing 4 irq 10 [ 52.373393] ipmi_si ipmi_si.0: Removing SMBIOS-specified kcs state machine in favor of ACPI [ 52.373394] ipmi_si: Adding ACPI-specified kcs state machine [ 52.373525] ipmi_si: Trying ACPI-specified kcs state machine at i/o address 0xca8, slave address 0x20, irq 10 [ 52.383249] ipmi_si IPI0001:00: The BMC does not support setting the recv irq bit, compensating, but the BMC needs to be fixed. [ 52.384339] ipmi_si IPI0001:00: Using irq 10 [ 52.386526] ipmi_si IPI0001:00: Found new BMC (man_id: 0x0002a2, prod_id: 0x0100, dev_id: 0x20) [ 52.451033] ipmi_si IPI0001:00: IPMI kcs interface initialized [ 52.530863] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.532651] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.534462] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.536247] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.538303] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.540619] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.542437] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.544326] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.546136] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.547929] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.550191] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.552025] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.554286] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.556497] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.558286] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.560073] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.561893] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.563702] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.565514] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.567358] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.569133] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.570922] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.573220] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.575048] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.576840] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.578630] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.580404] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.582184] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.583970] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.585748] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.587524] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.589346] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.591110] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.592877] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.594663] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.596455] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.598220] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.599993] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.601870] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.604404] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.606315] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.608150] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.610443] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.612406] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.614239] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.616070] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.617917] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.619737] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.621580] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.623418] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.625248] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.627103] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.628927] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.630756] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.632660] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.634440] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.636539] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.638316] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.640109] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 52.641885] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.082256] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.085842] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.089766] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.093291] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.096847] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.100415] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.103935] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.107705] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.111235] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.115094] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.119298] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.122825] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.126484] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.130642] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.134336] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.138008] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.141602] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.145120] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.148663] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.152159] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.155795] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.159295] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.162796] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.166308] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.170241] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.174258] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.177772] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.181365] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.184846] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.188500] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.192020] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.195895] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.199480] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.202998] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.206568] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.210105] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.213644] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.217141] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.220728] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.224258] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.227962] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.231513] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.235005] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.238503] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.241978] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.245582] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.249047] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.252569] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.256037] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.259536] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.263011] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.266512] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.270025] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.273535] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.277522] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.281735] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.285347] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.288910] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.292464] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.295972] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.797070] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.802324] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.809649] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.815188] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.820502] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.825817] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.831284] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.836557] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.841850] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.848707] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.854092] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.859742] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.864927] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.871907] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.877237] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.882712] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.887960] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.895573] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.900821] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.906101] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.911355] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.919842] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.925455] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.930840] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.936150] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.941962] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.947307] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.953083] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.960007] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.965294] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.970504] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.975765] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.981276] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.986556] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.991852] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 53.998570] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.004119] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.009812] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.015089] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.020645] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.025911] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.031156] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.036428] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.042226] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.047615] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.052952] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.058225] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.063479] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.069320] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.074493] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.079766] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.084998] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.092572] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.097999] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.103227] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.108457] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.114278] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.119704] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.125180] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.132088] mpt3sas_cm0: log_info(0x31200205): originator(PL), code(0x20), sub_code(0x0205) [ 54.611073] ses 0:0:0:0: Attached Enclosure device [ 54.611169] ses 0:0:61:0: Attached Enclosure device [ 54.611238] ses 0:0:122:0: Attached Enclosure device [ 54.611311] ses 0:0:183:0: Attached Enclosure device [ 54.611378] ses 0:0:244:0: Attached Enclosure device [ 56.752324] ccp 0000:02:00.2: 3 command queues available [ 56.752364] ccp 0000:02:00.2: irq 238 for MSI/MSI-X [ 56.752376] ccp 0000:02:00.2: irq 239 for MSI/MSI-X [ 56.752459] ccp 0000:02:00.2: Queue 2 can access 4 LSB regions [ 56.752461] ccp 0000:02:00.2: Queue 3 can access 4 LSB regions [ 56.752463] ccp 0000:02:00.2: Queue 4 can access 4 LSB regions [ 56.752465] ccp 0000:02:00.2: Queue 0 gets LSB 4 [ 56.752466] ccp 0000:02:00.2: Queue 1 gets LSB 5 [ 56.752467] ccp 0000:02:00.2: Queue 2 gets LSB 6 [ 56.754033] ccp 0000:02:00.2: enabled [ 56.754223] ccp 0000:03:00.1: 5 command queues available [ 56.754268] ccp 0000:03:00.1: irq 241 for MSI/MSI-X [ 56.754341] ccp 0000:03:00.1: Queue 0 can access 7 LSB regions [ 56.754343] ccp 0000:03:00.1: Queue 1 can access 7 LSB regions [ 56.754345] ccp 0000:03:00.1: Queue 2 can access 7 LSB regions [ 56.754347] ccp 0000:03:00.1: Queue 3 can access 7 LSB regions [ 56.754349] ccp 0000:03:00.1: Queue 4 can access 7 LSB regions [ 56.754351] ccp 0000:03:00.1: Queue 0 gets LSB 1 [ 56.754352] ccp 0000:03:00.1: Queue 1 gets LSB 2 [ 56.754353] ccp 0000:03:00.1: Queue 2 gets LSB 3 [ 56.754354] ccp 0000:03:00.1: Queue 3 gets LSB 4 [ 56.754355] ccp 0000:03:00.1: Queue 4 gets LSB 5 [ 56.760333] piix4_smbus 0000:00:14.0: SMBus Host Controller at 0xb00, revision 0 [ 56.760335] piix4_smbus 0000:00:14.0: Using register 0x2e for SMBus port selection [ 56.766518] ccp 0000:03:00.1: enabled [ 56.766715] ccp 0000:41:00.2: 3 command queues available [ 56.766754] ccp 0000:41:00.2: irq 243 for MSI/MSI-X [ 56.766763] ccp 0000:41:00.2: irq 244 for MSI/MSI-X [ 56.766872] ccp 0000:41:00.2: Queue 2 can access 4 LSB regions [ 56.766874] ccp 0000:41:00.2: Queue 3 can access 4 LSB regions [ 56.766876] ccp 0000:41:00.2: Queue 4 can access 4 LSB regions [ 56.766877] ccp 0000:41:00.2: Queue 0 gets LSB 4 [ 56.766878] ccp 0000:41:00.2: Queue 1 gets LSB 5 [ 56.766879] ccp 0000:41:00.2: Queue 2 gets LSB 6 [ 56.768358] ccp 0000:41:00.2: enabled [ 56.768537] ccp 0000:42:00.1: 5 command queues available [ 56.768566] ccp 0000:42:00.1: irq 246 for MSI/MSI-X [ 56.768643] ccp 0000:42:00.1: Queue 0 can access 7 LSB regions [ 56.768645] ccp 0000:42:00.1: Queue 1 can access 7 LSB regions [ 56.768647] ccp 0000:42:00.1: Queue 2 can access 7 LSB regions [ 56.768648] ccp 0000:42:00.1: Queue 3 can access 7 LSB regions [ 56.768650] ccp 0000:42:00.1: Queue 4 can access 7 LSB regions [ 56.768651] ccp 0000:42:00.1: Queue 0 gets LSB 1 [ 56.768652] ccp 0000:42:00.1: Queue 1 gets LSB 2 [ 56.768653] ccp 0000:42:00.1: Queue 2 gets LSB 3 [ 56.768654] ccp 0000:42:00.1: Queue 3 gets LSB 4 [ 56.768655] ccp 0000:42:00.1: Queue 4 gets LSB 5 [ 56.776648] ccp 0000:42:00.1: enabled [ 56.776902] ccp 0000:85:00.2: 3 command queues available [ 56.776942] ccp 0000:85:00.2: irq 248 for MSI/MSI-X [ 56.776964] ccp 0000:85:00.2: irq 249 for MSI/MSI-X [ 56.777061] ccp 0000:85:00.2: Queue 2 can access 4 LSB regions [ 56.777063] ccp 0000:85:00.2: Queue 3 can access 4 LSB regions [ 56.777065] ccp 0000:85:00.2: Queue 4 can access 4 LSB regions [ 56.777067] ccp 0000:85:00.2: Queue 0 gets LSB 4 [ 56.777069] ccp 0000:85:00.2: Queue 1 gets LSB 5 [ 56.777070] ccp 0000:85:00.2: Queue 2 gets LSB 6 [ 56.779117] ccp 0000:85:00.2: enabled [ 56.779303] ccp 0000:86:00.1: 5 command queues available [ 56.779349] ccp 0000:86:00.1: irq 251 for MSI/MSI-X [ 56.779415] ccp 0000:86:00.1: Queue 0 can access 7 LSB regions [ 56.779417] ccp 0000:86:00.1: Queue 1 can access 7 LSB regions [ 56.779419] ccp 0000:86:00.1: Queue 2 can access 7 LSB regions [ 56.779421] ccp 0000:86:00.1: Queue 3 can access 7 LSB regions [ 56.779423] ccp 0000:86:00.1: Queue 4 can access 7 LSB regions [ 56.779425] ccp 0000:86:00.1: Queue 0 gets LSB 1 [ 56.779426] ccp 0000:86:00.1: Queue 1 gets LSB 2 [ 56.779427] ccp 0000:86:00.1: Queue 2 gets LSB 3 [ 56.779428] ccp 0000:86:00.1: Queue 3 gets LSB 4 [ 56.779429] ccp 0000:86:00.1: Queue 4 gets LSB 5 [ 56.780932] ccp 0000:86:00.1: enabled [ 56.781219] ccp 0000:c2:00.2: 3 command queues available [ 56.781261] ccp 0000:c2:00.2: irq 253 for MSI/MSI-X [ 56.781273] ccp 0000:c2:00.2: irq 254 for MSI/MSI-X [ 56.781364] ccp 0000:c2:00.2: Queue 2 can access 4 LSB regions [ 56.781366] ccp 0000:c2:00.2: Queue 3 can access 4 LSB regions [ 56.781368] ccp 0000:c2:00.2: Queue 4 can access 4 LSB regions [ 56.781370] ccp 0000:c2:00.2: Queue 0 gets LSB 4 [ 56.781371] ccp 0000:c2:00.2: Queue 1 gets LSB 5 [ 56.781374] ccp 0000:c2:00.2: Queue 2 gets LSB 6 [ 56.789713] ccp 0000:c2:00.2: enabled [ 56.791715] ccp 0000:c3:00.1: 5 command queues available [ 56.791752] ccp 0000:c3:00.1: irq 256 for MSI/MSI-X [ 56.791790] ccp 0000:c3:00.1: Queue 0 can access 7 LSB regions [ 56.791792] ccp 0000:c3:00.1: Queue 1 can access 7 LSB regions [ 56.791794] ccp 0000:c3:00.1: Queue 2 can access 7 LSB regions [ 56.791796] ccp 0000:c3:00.1: Queue 3 can access 7 LSB regions [ 56.791798] ccp 0000:c3:00.1: Queue 4 can access 7 LSB regions [ 56.791800] ccp 0000:c3:00.1: Queue 0 gets LSB 1 [ 56.791801] ccp 0000:c3:00.1: Queue 1 gets LSB 2 [ 56.791802] ccp 0000:c3:00.1: Queue 2 gets LSB 3 [ 56.791803] ccp 0000:c3:00.1: Queue 3 gets LSB 4 [ 56.791804] ccp 0000:c3:00.1: Queue 4 gets LSB 5 [ 56.793117] ccp 0000:c3:00.1: enabled [ 57.001128] input: PC Speaker as /devices/platform/pcspkr/input/input2 [ 58.136388] dcdbas dcdbas: Dell Systems Management Base Driver (version 5.6.0-3.3) [ 58.448587] cryptd: max_cpu_qlen set to 1000 [ 58.470770] AVX2 version of gcm_enc/dec engaged. [ 58.475393] AES CTR mode by8 optimization enabled [ 58.486009] alg: No test for __gcm-aes-aesni (__driver-gcm-aes-aesni) [ 58.492895] alg: No test for __generic-gcm-aes-aesni (__driver-generic-gcm-aes-aesni) [ 58.560013] kvm: Nested Paging enabled [ 58.567570] MCE: In-kernel MCE decoding enabled. [ 58.576430] AMD64 EDAC driver v3.4.0 [ 58.580045] EDAC amd64: DRAM ECC enabled. [ 58.584064] EDAC amd64: F17h detected (node 0). [ 58.588648] EDAC MC: UMC0 chip selects: [ 58.588651] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.593358] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.598066] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.602785] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.607561] EDAC MC: UMC1 chip selects: [ 58.607592] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.612326] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.617048] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.621769] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.626489] EDAC amd64: using x8 syndromes. [ 58.630685] EDAC amd64: MCT channel count: 2 [ 58.635212] EDAC MC0: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:18.3 [ 58.642631] EDAC amd64: DRAM ECC enabled. [ 58.646658] EDAC amd64: F17h detected (node 1). [ 58.651245] EDAC MC: UMC0 chip selects: [ 58.651248] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.655962] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.660679] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.665390] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.670110] EDAC MC: UMC1 chip selects: [ 58.670112] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.674822] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.679533] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.684246] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.688961] EDAC amd64: using x8 syndromes. [ 58.693152] EDAC amd64: MCT channel count: 2 [ 58.697849] EDAC MC1: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:19.3 [ 58.705260] EDAC amd64: DRAM ECC enabled. [ 58.709278] EDAC amd64: F17h detected (node 2). [ 58.713870] EDAC MC: UMC0 chip selects: [ 58.713873] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.718590] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.723304] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.728021] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.732739] EDAC MC: UMC1 chip selects: [ 58.732743] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.737455] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.742176] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.746887] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.751601] EDAC amd64: using x8 syndromes. [ 58.755804] EDAC amd64: MCT channel count: 2 [ 58.760821] EDAC MC2: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1a.3 [ 58.768296] EDAC amd64: DRAM ECC enabled. [ 58.772336] EDAC amd64: F17h detected (node 3). [ 58.776919] EDAC MC: UMC0 chip selects: [ 58.776922] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.781629] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.786337] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.791058] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.795772] EDAC MC: UMC1 chip selects: [ 58.795776] EDAC amd64: MC: 0: 0MB 1: 0MB [ 58.800497] EDAC amd64: MC: 2: 32767MB 3: 32767MB [ 58.805215] EDAC amd64: MC: 4: 0MB 5: 0MB [ 58.809939] EDAC amd64: MC: 6: 0MB 7: 0MB [ 58.814662] EDAC amd64: using x8 syndromes. [ 58.818869] EDAC amd64: MCT channel count: 2 [ 58.824302] EDAC MC3: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1b.3 [ 58.831854] EDAC PCI0: Giving out device to module 'amd64_edac' controller 'EDAC PCI controller': DEV '0000:00:18.0' (POLLED) [ 65.377204] device-mapper: multipath service-time: version 0.3.0 loaded [ 103.233051] Adding 4194300k swap on /dev/sdig3. Priority:-2 extents:1 across:4194300k FS [ 103.275868] type=1305 audit(1547777804.777:3): audit_pid=64442 old=0 auid=4294967295 ses=4294967295 res=1 [ 103.297051] RPC: Registered named UNIX socket transport module. [ 103.303068] RPC: Registered udp transport module. [ 103.309160] RPC: Registered tcp transport module. [ 103.315251] RPC: Registered tcp NFSv4.1 backchannel transport module. [ 103.634463] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 103.713429] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 103.768306] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 103.897675] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 103.946294] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 104.029289] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 104.043437] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 104.059621] mlx5_core 0000:84:00.0: slow_pci_heuristic:5202:(pid 64735): Max link speed = 100000, PCI BW = 126016 [ 104.069960] mlx5_core 0000:84:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(64) RxCqeCmprss(0) [ 104.078160] mlx5_core 0000:84:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(64) RxCqeCmprss(0) [ 104.211712] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 104.224671] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 104.275862] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 104.324098] Request for unknown module key 'Mellanox Technologies signing key: 61feb074fc7292f958419386ffdd9d5ca999e403' err -11 [ 104.613396] tg3 0000:81:00.0: irq 257 for MSI/MSI-X [ 104.613409] tg3 0000:81:00.0: irq 258 for MSI/MSI-X [ 104.613430] tg3 0000:81:00.0: irq 259 for MSI/MSI-X [ 104.613441] tg3 0000:81:00.0: irq 260 for MSI/MSI-X [ 104.613451] tg3 0000:81:00.0: irq 261 for MSI/MSI-X [ 104.739552] IPv6: ADDRCONF(NETDEV_UP): em1: link is not ready [ 108.250404] tg3 0000:81:00.0 em1: Link is up at 1000 Mbps, full duplex [ 108.256939] tg3 0000:81:00.0 em1: Flow control is on for TX and on for RX [ 108.263731] tg3 0000:81:00.0 em1: EEE is enabled [ 108.268366] IPv6: ADDRCONF(NETDEV_CHANGE): em1: link becomes ready [ 108.974858] IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready [ 109.087753] IPv6: ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready [ 113.221915] FS-Cache: Loaded [ 113.251329] FS-Cache: Netfs 'nfs' registered for caching [ 113.260286] Key type dns_resolver registered [ 113.288387] NFS: Registering the id_resolver key type [ 113.293922] Key type id_resolver registered [ 113.298191] Key type id_legacy registered [ 3653.228655] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [ 3653.236433] alg: No test for adler32 (adler32-zlib) [ 3654.040378] Lustre: Lustre: Build Version: 2.12.0 [ 3654.160641] LNet: Using FastReg for registration [ 3654.177597] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [ 3720.148401] md: md7 stopped. [ 3720.168343] async_tx: api initialized (async) [ 3720.174688] xor: automatically using best checksumming function: [ 3720.189898] avx : 9568.000 MB/sec [ 3720.222908] raid6: sse2x1 gen() 6093 MB/s [ 3720.243899] raid6: sse2x2 gen() 11335 MB/s [ 3720.264902] raid6: sse2x4 gen() 12980 MB/s [ 3720.285899] raid6: avx2x1 gen() 14285 MB/s [ 3720.306902] raid6: avx2x2 gen() 18871 MB/s [ 3720.327903] raid6: avx2x4 gen() 18875 MB/s [ 3720.332173] raid6: using algorithm avx2x4 gen() (18875 MB/s) [ 3720.337837] raid6: using avx2x2 recovery algorithm [ 3720.359200] md/raid:md7: device dm-72 operational as raid disk 0 [ 3720.365225] md/raid:md7: device dm-107 operational as raid disk 9 [ 3720.371333] md/raid:md7: device dm-106 operational as raid disk 8 [ 3720.377435] md/raid:md7: device dm-93 operational as raid disk 7 [ 3720.383474] md/raid:md7: device dm-92 operational as raid disk 6 [ 3720.389488] md/raid:md7: device dm-80 operational as raid disk 5 [ 3720.395523] md/raid:md7: device dm-79 operational as raid disk 4 [ 3720.401543] md/raid:md7: device dm-67 operational as raid disk 3 [ 3720.407582] md/raid:md7: device dm-66 operational as raid disk 2 [ 3720.413597] md/raid:md7: device dm-83 operational as raid disk 1 [ 3720.420644] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3720.466020] md7: detected capacity change from 0 to 64011422924800 [ 3720.484985] md: md9 stopped. [ 3720.504430] md/raid:md9: device dm-116 operational as raid disk 0 [ 3720.510618] md/raid:md9: device dm-111 operational as raid disk 9 [ 3720.516719] md/raid:md9: device dm-110 operational as raid disk 8 [ 3720.522820] md/raid:md9: device dm-98 operational as raid disk 7 [ 3720.528836] md/raid:md9: device dm-97 operational as raid disk 6 [ 3720.534852] md/raid:md9: device dm-85 operational as raid disk 5 [ 3720.540866] md/raid:md9: device dm-84 operational as raid disk 4 [ 3720.546890] md/raid:md9: device dm-71 operational as raid disk 3 [ 3720.552940] md/raid:md9: device dm-70 operational as raid disk 2 [ 3720.558993] md/raid:md9: device dm-117 operational as raid disk 1 [ 3720.566038] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3720.595960] md9: detected capacity change from 0 to 64011422924800 [ 3720.612654] md: md11 stopped. [ 3720.641693] md/raid:md11: device dm-62 operational as raid disk 0 [ 3720.647821] md/raid:md11: device dm-115 operational as raid disk 9 [ 3720.654018] md/raid:md11: device dm-114 operational as raid disk 8 [ 3720.660211] md/raid:md11: device dm-102 operational as raid disk 7 [ 3720.666409] md/raid:md11: device dm-101 operational as raid disk 6 [ 3720.672609] md/raid:md11: device dm-89 operational as raid disk 5 [ 3720.678717] md/raid:md11: device dm-88 operational as raid disk 4 [ 3720.684828] md/raid:md11: device dm-76 operational as raid disk 3 [ 3720.690939] md/raid:md11: device dm-75 operational as raid disk 2 [ 3720.697078] md/raid:md11: device dm-63 operational as raid disk 1 [ 3720.704485] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3720.732257] md11: detected capacity change from 0 to 64011422924800 [ 3720.748794] md: md1 stopped. [ 3720.771018] md/raid:md1: device dm-12 operational as raid disk 0 [ 3720.777044] md/raid:md1: device dm-47 operational as raid disk 9 [ 3720.783066] md/raid:md1: device dm-46 operational as raid disk 8 [ 3720.789084] md/raid:md1: device dm-33 operational as raid disk 7 [ 3720.795103] md/raid:md1: device dm-32 operational as raid disk 6 [ 3720.801124] md/raid:md1: device dm-20 operational as raid disk 5 [ 3720.807140] md/raid:md1: device dm-19 operational as raid disk 4 [ 3720.813155] md/raid:md1: device dm-7 operational as raid disk 3 [ 3720.819077] md/raid:md1: device dm-6 operational as raid disk 2 [ 3720.825008] md/raid:md1: device dm-23 operational as raid disk 1 [ 3720.831727] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3720.861751] md1: detected capacity change from 0 to 64011422924800 [ 3720.873234] md: md3 stopped. [ 3720.900951] md/raid:md3: device dm-56 operational as raid disk 0 [ 3720.906974] md/raid:md3: device dm-51 operational as raid disk 9 [ 3720.912988] md/raid:md3: device dm-50 operational as raid disk 8 [ 3720.918999] md/raid:md3: device dm-38 operational as raid disk 7 [ 3720.925013] md/raid:md3: device dm-37 operational as raid disk 6 [ 3720.931029] md/raid:md3: device dm-25 operational as raid disk 5 [ 3720.937042] md/raid:md3: device dm-24 operational as raid disk 4 [ 3720.943055] md/raid:md3: device dm-11 operational as raid disk 3 [ 3720.949066] md/raid:md3: device dm-10 operational as raid disk 2 [ 3720.955079] md/raid:md3: device dm-57 operational as raid disk 1 [ 3720.961949] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3720.991145] md3: detected capacity change from 0 to 64011422924800 [ 3721.003162] md: md5 stopped. [ 3721.061136] md/raid:md5: device dm-2 operational as raid disk 0 [ 3721.067105] md/raid:md5: device dm-55 operational as raid disk 9 [ 3721.073205] md/raid:md5: device dm-54 operational as raid disk 8 [ 3721.079293] md/raid:md5: device dm-42 operational as raid disk 7 [ 3721.085332] md/raid:md5: device dm-41 operational as raid disk 6 [ 3721.091363] md/raid:md5: device dm-29 operational as raid disk 5 [ 3721.097394] md/raid:md5: device dm-28 operational as raid disk 4 [ 3721.103417] md/raid:md5: device dm-16 operational as raid disk 3 [ 3721.109450] md/raid:md5: device dm-15 operational as raid disk 2 [ 3721.115483] md/raid:md5: device dm-3 operational as raid disk 1 [ 3721.123933] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 3721.163716] md5: detected capacity change from 0 to 64011422924800 [ 3721.268159] LDISKFS-fs warning (device md7): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3721.722847] LDISKFS-fs warning (device md9): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3721.756001] LDISKFS-fs warning (device md11): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3721.993486] LDISKFS-fs warning (device md1): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3722.221699] LDISKFS-fs warning (device md3): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3722.487078] LDISKFS-fs warning (device md5): ldiskfs_multi_mount_protect:321: MMP interval 42 higher than expected, please wait. [ 3763.347180] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 3763.799170] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 3763.832014] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 3764.073183] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 3764.197926] LDISKFS-fs (md7): recovery complete [ 3764.229190] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3764.303787] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 3764.567201] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 3764.599865] LDISKFS-fs (md9): recovery complete [ 3764.625008] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3764.704427] LDISKFS-fs (md11): recovery complete [ 3764.726033] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3764.992097] LDISKFS-fs (md1): recovery complete [ 3765.020530] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3765.161960] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 17 seconds [ 3765.324739] LDISKFS-fs (md3): recovery complete [ 3765.348460] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3765.363844] LDISKFS-fs (md5): recovery complete [ 3765.385220] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 3768.162027] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [ 3770.162075] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 22 seconds [ 3770.172335] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [ 3771.164295] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [ 3771.196180] Lustre: fir-OST0021: in recovery but waiting for the first client to connect [ 3773.162149] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 25 seconds [ 3773.172402] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [ 3776.164186] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [ 3776.173007] Lustre: fir-OST001f: in recovery but waiting for the first client to connect [ 3777.270730] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3777.271129] Lustre: fir-OST0021: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3777.271167] Lustre: fir-OST0021: Connection restored to (at 10.8.9.6@o2ib6) [ 3777.304005] LustreError: Skipped 4 previous similar messages [ 3778.162259] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 30 seconds [ 3778.172513] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [ 3781.164483] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [ 3781.177099] Lustre: fir-OST0023: in recovery but waiting for the first client to connect [ 3783.566003] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.8.6.4@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3783.566063] Lustre: fir-OST001f: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3783.566298] Lustre: fir-OST0021: Connection restored to (at 10.8.6.4@o2ib6) [ 3783.599292] LustreError: Skipped 3 previous similar messages [ 3784.677035] Lustre: fir-OST001f: Connection restored to (at 10.8.9.6@o2ib6) [ 3784.677185] Lustre: fir-OST0023: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3784.693218] Lustre: Skipped 2 previous similar messages [ 3787.162468] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 39 seconds [ 3787.172726] LNet: 121912:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [ 3787.184155] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [ 3787.192987] Lustre: fir-OST0019: in recovery but waiting for the first client to connect [ 3787.295305] Lustre: fir-OST0023: Connection restored to (at 10.8.6.4@o2ib6) [ 3787.295403] Lustre: fir-OST0019: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3787.311517] Lustre: Skipped 1 previous similar message [ 3792.164675] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [ 3792.176795] Lustre: fir-OST001d: in recovery but waiting for the first client to connect [ 3793.681501] Lustre: fir-OST0019: Connection restored to (at 10.8.9.6@o2ib6) [ 3793.688626] Lustre: fir-OST001d: Will be in recovery for at least 5:00, or until 7 clients reconnect [ 3804.976741] Lustre: fir-OST001b: Connection restored to (at 10.8.6.4@o2ib6) [ 3804.983801] Lustre: Skipped 9 previous similar messages [ 3860.644807] Lustre: fir-OST0021: Connection restored to (at 10.0.10.51@o2ib7) [ 3860.652038] Lustre: Skipped 15 previous similar messages [ 3860.839844] Lustre: fir-OST001f: Recovery over after 1:18, of 7 clients 7 recovered and 0 were evicted. [ 3860.849254] Lustre: Skipped 2 previous similar messages [ 3903.665904] Lustre: fir-OST001f: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [ 3903.675130] Lustre: Skipped 7 previous similar messages [ 3965.401002] Lustre: fir-OST0021: deleting orphan objects from 0x0:34780 to 0x0:34849 [ 3965.404211] Lustre: fir-OST0019: deleting orphan objects from 0x0:34780 to 0x0:34849 [ 3965.405050] Lustre: fir-OST001d: deleting orphan objects from 0x0:34780 to 0x0:34849 [ 3965.407700] Lustre: fir-OST0023: deleting orphan objects from 0x0:34780 to 0x0:34849 [ 3965.408381] Lustre: fir-OST001b: deleting orphan objects from 0x0:34780 to 0x0:34849 [ 3965.419266] Lustre: fir-OST001f: deleting orphan objects from 0x0:34780 to 0x0:34849 [ 4158.401438] Lustre: fir-OST001d: Connection restored to 30ed9acf-7f48-0f0d-1126-afaebb670d13 (at 10.0.10.3@o2ib7) [ 4158.411712] Lustre: Skipped 8 previous similar messages [ 4273.802844] LNetError: 244:0:(o2iblnd_cb.c:2935:kiblnd_rejected()) 10.0.10.52@o2ib7 rejected: o2iblnd fatal error [ 4273.813132] Lustre: 122709:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1547781975/real 1547781975] req@ffff9283b50e8900 x1622966382110192/t0(0) o400->fir-MDT0001-lwp-OST0021@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781982 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [ 4273.813158] Lustre: fir-MDT0003-lwp-OST0021: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [ 4273.857944] Lustre: 122709:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 10 previous similar messages [ 4280.801860] Lustre: 122704:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547781975/real 1547781975] req@ffff9283b50e8600 x1622966382110496/t0(0) o400->fir-MDT0000-lwp-OST001b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781982 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [ 4280.801863] Lustre: 122700:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547781975/real 1547781975] req@ffff9283b50e8f00 x1622966382110432/t0(0) o400->fir-MDT0000-lwp-OST001d@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781982 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [ 4280.801866] Lustre: 122700:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [ 4280.801869] Lustre: fir-MDT0000-lwp-OST0019: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [ 4280.801872] Lustre: Skipped 11 previous similar messages [ 4280.889892] Lustre: 122704:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [ 4282.479900] Lustre: 982:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547781977/real 1547781977] req@ffff927448fc0000 x1622966382110800/t0(0) o39->fir-MDT0002-lwp-OST001b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547781983 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [ 4282.487208] Lustre: Failing over fir-OST001d [ 4282.512286] Lustre: 982:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [ 4283.285720] Lustre: server umount fir-OST0023 complete [ 4283.290873] Lustre: Skipped 3 previous similar messages [ 4283.471681] md5: detected capacity change from 64011422924800 to 0 [ 4283.477874] md: md5 stopped. [ 4283.619515] md9: detected capacity change from 64011422924800 to 0 [ 4283.625724] md: md9 stopped. [ 4283.636161] md1: detected capacity change from 64011422924800 to 0 [ 4283.642377] md: md1 stopped. [ 4283.690796] md7: detected capacity change from 64011422924800 to 0 [ 4283.696992] md: md7 stopped. [ 4283.811044] md11: detected capacity change from 64011422924800 to 0 [ 4283.817325] md: md11 stopped. [ 4283.817534] md3: detected capacity change from 64011422924800 to 0 [ 4283.817537] md: md3 stopped. [ 4286.053883] LNetError: 850:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.209@o2ib7 on NA (ib0:1:10.0.10.106): bad dst nid 10.0.10.106@o2ib7 [ 4287.205007] LNet: Removed LNI 10.0.10.106@o2ib7 [ 4321.576520] md: md9 stopped. [ 4321.611123] md/raid:md9: device dm-116 operational as raid disk 0 [ 4321.617229] md/raid:md9: device dm-111 operational as raid disk 9 [ 4321.623334] md/raid:md9: device dm-110 operational as raid disk 8 [ 4321.629434] md/raid:md9: device dm-98 operational as raid disk 7 [ 4321.635448] md/raid:md9: device dm-97 operational as raid disk 6 [ 4321.641462] md/raid:md9: device dm-85 operational as raid disk 5 [ 4321.647468] md/raid:md9: device dm-84 operational as raid disk 4 [ 4321.653474] md/raid:md9: device dm-71 operational as raid disk 3 [ 4321.659481] md/raid:md9: device dm-70 operational as raid disk 2 [ 4321.665495] md/raid:md9: device dm-117 operational as raid disk 1 [ 4321.672439] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4321.690756] md9: detected capacity change from 0 to 64011422924800 [ 4321.710110] md: md1 stopped. [ 4321.752042] md/raid:md1: device dm-12 operational as raid disk 0 [ 4321.758071] md/raid:md1: device dm-47 operational as raid disk 9 [ 4321.764097] md/raid:md1: device dm-46 operational as raid disk 8 [ 4321.770109] md/raid:md1: device dm-33 operational as raid disk 7 [ 4321.776122] md/raid:md1: device dm-32 operational as raid disk 6 [ 4321.782138] md/raid:md1: device dm-20 operational as raid disk 5 [ 4321.788154] md/raid:md1: device dm-19 operational as raid disk 4 [ 4321.794174] md/raid:md1: device dm-7 operational as raid disk 3 [ 4321.800110] md/raid:md1: device dm-6 operational as raid disk 2 [ 4321.806044] md/raid:md1: device dm-23 operational as raid disk 1 [ 4321.813132] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4321.831156] md1: detected capacity change from 0 to 64011422924800 [ 4321.847212] md: md5 stopped. [ 4321.887113] md/raid:md5: device dm-2 operational as raid disk 0 [ 4321.893042] md/raid:md5: device dm-55 operational as raid disk 9 [ 4321.899054] md/raid:md5: device dm-54 operational as raid disk 8 [ 4321.905071] md/raid:md5: device dm-42 operational as raid disk 7 [ 4321.911083] md/raid:md5: device dm-41 operational as raid disk 6 [ 4321.917098] md/raid:md5: device dm-29 operational as raid disk 5 [ 4321.923113] md/raid:md5: device dm-28 operational as raid disk 4 [ 4321.929127] md/raid:md5: device dm-16 operational as raid disk 3 [ 4321.935142] md/raid:md5: device dm-15 operational as raid disk 2 [ 4321.941157] md/raid:md5: device dm-3 operational as raid disk 1 [ 4321.948341] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4321.967276] md5: detected capacity change from 0 to 64011422924800 [ 4321.977615] md: md11 stopped. [ 4322.036001] md/raid:md11: device dm-62 operational as raid disk 0 [ 4322.042115] md/raid:md11: device dm-115 operational as raid disk 9 [ 4322.048313] md/raid:md11: device dm-114 operational as raid disk 8 [ 4322.054510] md/raid:md11: device dm-102 operational as raid disk 7 [ 4322.060703] md/raid:md11: device dm-101 operational as raid disk 6 [ 4322.066891] md/raid:md11: device dm-89 operational as raid disk 5 [ 4322.073005] md/raid:md11: device dm-88 operational as raid disk 4 [ 4322.079124] md/raid:md11: device dm-76 operational as raid disk 3 [ 4322.085244] md/raid:md11: device dm-75 operational as raid disk 2 [ 4322.091355] md/raid:md11: device dm-63 operational as raid disk 1 [ 4322.098496] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4322.130581] md11: detected capacity change from 0 to 64011422924800 [ 4322.147397] md: md7 stopped. [ 4322.184670] md/raid:md7: device dm-72 operational as raid disk 0 [ 4322.190691] md/raid:md7: device dm-107 operational as raid disk 9 [ 4322.196805] md/raid:md7: device dm-106 operational as raid disk 8 [ 4322.202916] md/raid:md7: device dm-93 operational as raid disk 7 [ 4322.208931] md/raid:md7: device dm-92 operational as raid disk 6 [ 4322.214955] md/raid:md7: device dm-80 operational as raid disk 5 [ 4322.220977] md/raid:md7: device dm-79 operational as raid disk 4 [ 4322.226995] md/raid:md7: device dm-67 operational as raid disk 3 [ 4322.233018] md/raid:md7: device dm-66 operational as raid disk 2 [ 4322.239038] md/raid:md7: device dm-83 operational as raid disk 1 [ 4322.245821] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4322.268189] md7: detected capacity change from 0 to 64011422924800 [ 4322.284401] md: md3 stopped. [ 4322.309115] md/raid:md3: device dm-56 operational as raid disk 0 [ 4322.315137] md/raid:md3: device dm-51 operational as raid disk 9 [ 4322.321157] md/raid:md3: device dm-50 operational as raid disk 8 [ 4322.327174] md/raid:md3: device dm-38 operational as raid disk 7 [ 4322.333192] md/raid:md3: device dm-37 operational as raid disk 6 [ 4322.339205] md/raid:md3: device dm-25 operational as raid disk 5 [ 4322.345217] md/raid:md3: device dm-24 operational as raid disk 4 [ 4322.351234] md/raid:md3: device dm-11 operational as raid disk 3 [ 4322.357249] md/raid:md3: device dm-10 operational as raid disk 2 [ 4322.363264] md/raid:md3: device dm-57 operational as raid disk 1 [ 4322.369992] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 4322.398608] md3: detected capacity change from 0 to 64011422924800 [ 4343.084450] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 4343.395338] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4344.462486] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 4344.776189] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4345.053502] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 4345.369826] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4345.742513] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 4346.054194] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4350.949641] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 4351.262320] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4357.941931] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 4358.259143] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4373.787123] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [ 4373.794679] alg: No test for adler32 (adler32-zlib) [ 4374.600792] Lustre: Lustre: Build Version: 2.12.0 [ 4374.729098] LNet: Using FastReg for registration [ 4374.744679] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [ 4379.830347] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 4380.087348] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 4380.165596] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4380.361991] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 4380.433781] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4380.671309] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [ 4380.701891] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4380.798309] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 4380.837476] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 4380.968439] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4381.097811] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4381.115844] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 4381.173621] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4381.446917] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4381.503358] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 4381.754346] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 4381.769323] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [ 4381.836465] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [ 4381.953862] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.101.59@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4381.971239] LustreError: Skipped 5 previous similar messages [ 4382.054334] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 4382.060362] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4382.071830] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4382.228317] Lustre: fir-OST0021: Not available for connect from 10.0.10.52@o2ib7 (not set up) [ 4382.229977] Lustre: fir-OST0021: new disk, initializing [ 4382.338353] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 4382.371728] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4383.229330] Lustre: srv-fir-OST0021: No data found on store. Initialize space [ 4383.275682] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [ 4383.285240] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 4383.546848] Lustre: fir-OST001d: new disk, initializing [ 4383.882272] Lustre: srv-fir-OST0023: No data found on store. Initialize space [ 4383.889410] Lustre: Skipped 1 previous similar message [ 4383.903797] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [ 4383.911981] Lustre: Skipped 1 previous similar message [ 4384.122589] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4384.139977] LustreError: Skipped 5 previous similar messages [ 4384.701102] Lustre: fir-OST001f: new disk, initializing [ 4384.706339] Lustre: Skipped 3 previous similar messages [ 4385.526328] Lustre: fir-OST0021: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [ 4385.603770] Lustre: cli-fir-OST001d-super: Allocated super-sequence [0x0000000600000400-0x0000000640000400]:1d:ost] [ 4385.614206] Lustre: Skipped 1 previous similar message [ 4390.593491] Lustre: fir-OST0021: Connection restored to (at 10.0.10.52@o2ib7) [ 4390.600740] Lustre: Skipped 12 previous similar messages [ 4391.943649] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4391.960765] LustreError: Skipped 4 previous similar messages [ 4399.966809] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.6.4@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 4399.983921] LustreError: Skipped 4 previous similar messages [ 4404.466684] Lustre: fir-OST0023: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [ 4404.476876] Lustre: Skipped 15 previous similar messages [ 4406.954639] Lustre: fir-OST0019: Connection restored to (at 10.9.101.59@o2ib4) [ 4406.961955] Lustre: Skipped 5 previous similar messages [ 4416.944412] Lustre: fir-OST0019: Connection restored to (at 10.9.0.2@o2ib4) [ 4416.951476] Lustre: Skipped 6 previous similar messages [ 4424.967477] Lustre: fir-OST0019: Connection restored to (at 10.8.6.4@o2ib6) [ 4424.974534] Lustre: Skipped 10 previous similar messages [ 4565.867045] Lustre: fir-OST0019: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [ 4565.877310] Lustre: Skipped 5 previous similar messages [ 5037.208049] Lustre: fir-OST001f: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [ 5037.218319] Lustre: Skipped 4 previous similar messages [ 5717.667375] Lustre: fir-OST001f: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [ 6523.104305] Lustre: fir-OST0019: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [ 6523.114568] Lustre: Skipped 10 previous similar messages [12516.314615] Lustre: fir-OST0023: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [53820.074604] Lustre: fir-OST0023: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [55054.610648] Lustre: fir-OST001f: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [56078.678874] Lustre: fir-OST0019: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [57367.058494] Lustre: fir-OST0019: Connection restored to 0a11f947-203c-14dd-afce-1f3f31ad7212 (at 10.8.6.33@o2ib6) [57367.068764] Lustre: Skipped 10 previous similar messages [57774.485039] Lustre: 1456:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547835468/real 1547835468] req@ffff9283e90a3300 x1622967138984368/t0(0) o39->fir-MDT0002-lwp-OST0021@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835474 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [57774.485042] Lustre: 1452:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547835468/real 1547835468] req@ffff92744a398000 x1622967138984384/t0(0) o39->fir-MDT0002-lwp-OST0019@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835474 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [57774.485047] Lustre: 1452:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [57774.551087] Lustre: 1456:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [57774.961054] LNet: 5017:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [57774.971123] Lustre: fir-MDT0003-lwp-OST0019: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [57774.987104] Lustre: Skipped 6 previous similar messages [57774.992345] Lustre: 5056:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1547835473/real 1547835475] req@ffff92746779f800 x1622967138984464/t0(0) o400->fir-MDT0003-lwp-OST0021@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835480 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [57775.021016] Lustre: 5056:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [57775.961076] LNet: 5017:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 298 seconds [57775.971251] LNet: 5017:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [57775.980536] Lustre: fir-MDT0001-lwp-OST001f: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [57775.996547] Lustre: Skipped 4 previous similar messages [57780.486173] Lustre: 1454:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547835474/real 1547835474] req@ffff9263cf744200 x1622967138984752/t0(0) o39->fir-MDT0000-lwp-OST001f@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547835480 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [57780.491621] Lustre: Failing over fir-OST0023 [57780.518711] Lustre: 1454:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 8 previous similar messages [57780.900261] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [57780.917390] LustreError: Skipped 2 previous similar messages [57781.250655] Lustre: server umount fir-OST0019 complete [57781.893567] Lustre: server umount fir-OST001f complete [57781.898736] Lustre: Skipped 4 previous similar messages [57782.040779] md1: detected capacity change from 64011422924800 to 0 [57782.046979] md: md1 stopped. [57782.063454] md9: detected capacity change from 64011422924800 to 0 [57782.069652] md: md9 stopped. [57782.152450] md5: detected capacity change from 64011422924800 to 0 [57782.158650] md: md5 stopped. [57782.187781] md11: detected capacity change from 64011422924800 to 0 [57782.194067] md: md11 stopped. [57782.204081] md3: detected capacity change from 64011422924800 to 0 [57782.210339] md: md3 stopped. [57782.213470] md7: detected capacity change from 64011422924800 to 0 [57782.219670] md: md7 stopped. [57785.990304] LNet: Removed LNI 10.0.10.106@o2ib7 [57812.536091] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [57812.543673] alg: No test for adler32 (adler32-zlib) [57813.347755] Lustre: Lustre: Build Version: 2.12.0 [57813.469953] LNet: Using FastReg for registration [57813.486189] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [57819.548506] md: md9 stopped. [57819.558691] md/raid:md9: device dm-116 operational as raid disk 0 [57819.564793] md/raid:md9: device dm-111 operational as raid disk 9 [57819.570896] md/raid:md9: device dm-110 operational as raid disk 8 [57819.576996] md/raid:md9: device dm-98 operational as raid disk 7 [57819.583037] md/raid:md9: device dm-97 operational as raid disk 6 [57819.589045] md/raid:md9: device dm-85 operational as raid disk 5 [57819.595078] md/raid:md9: device dm-84 operational as raid disk 4 [57819.601089] md/raid:md9: device dm-71 operational as raid disk 3 [57819.607097] md/raid:md9: device dm-70 operational as raid disk 2 [57819.613111] md/raid:md9: device dm-117 operational as raid disk 1 [57819.619984] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [57819.637884] md9: detected capacity change from 0 to 64011422924800 [57819.652720] md: md7 stopped. [57819.673500] md/raid:md7: device dm-72 operational as raid disk 0 [57819.679522] md/raid:md7: device dm-107 operational as raid disk 9 [57819.685619] md/raid:md7: device dm-106 operational as raid disk 8 [57819.691719] md/raid:md7: device dm-93 operational as raid disk 7 [57819.697728] md/raid:md7: device dm-92 operational as raid disk 6 [57819.703741] md/raid:md7: device dm-80 operational as raid disk 5 [57819.709748] md/raid:md7: device dm-79 operational as raid disk 4 [57819.715762] md/raid:md7: device dm-67 operational as raid disk 3 [57819.721768] md/raid:md7: device dm-66 operational as raid disk 2 [57819.727773] md/raid:md7: device dm-83 operational as raid disk 1 [57819.734505] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [57819.752691] md7: detected capacity change from 0 to 64011422924800 [57819.777438] md: md1 stopped. [57819.802451] md/raid:md1: device dm-12 operational as raid disk 0 [57819.808469] md/raid:md1: device dm-47 operational as raid disk 9 [57819.814495] md/raid:md1: device dm-46 operational as raid disk 8 [57819.820516] md/raid:md1: device dm-33 operational as raid disk 7 [57819.826531] md/raid:md1: device dm-32 operational as raid disk 6 [57819.832545] md/raid:md1: device dm-20 operational as raid disk 5 [57819.838561] md/raid:md1: device dm-19 operational as raid disk 4 [57819.844586] md/raid:md1: device dm-7 operational as raid disk 3 [57819.850518] md/raid:md1: device dm-6 operational as raid disk 2 [57819.856477] md/raid:md1: device dm-23 operational as raid disk 1 [57819.863531] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [57819.882548] md1: detected capacity change from 0 to 64011422924800 [57819.894462] md: md3 stopped. [57819.956276] md/raid:md3: device dm-56 operational as raid disk 0 [57819.962309] md/raid:md3: device dm-51 operational as raid disk 9 [57819.968347] md/raid:md3: device dm-50 operational as raid disk 8 [57819.974449] md/raid:md3: device dm-38 operational as raid disk 7 [57819.980477] md/raid:md3: device dm-37 operational as raid disk 6 [57819.986496] md/raid:md3: device dm-25 operational as raid disk 5 [57819.992560] md/raid:md3: device dm-24 operational as raid disk 4 [57819.998635] md/raid:md3: device dm-11 operational as raid disk 3 [57820.004696] md/raid:md3: device dm-10 operational as raid disk 2 [57820.010736] md/raid:md3: device dm-57 operational as raid disk 1 [57820.017969] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [57820.037019] md3: detected capacity change from 0 to 64011422924800 [57820.059490] md: md11 stopped. [57820.085945] md/raid:md11: device dm-62 operational as raid disk 0 [57820.092080] md/raid:md11: device dm-115 operational as raid disk 9 [57820.098275] md/raid:md11: device dm-114 operational as raid disk 8 [57820.104463] md/raid:md11: device dm-102 operational as raid disk 7 [57820.110657] md/raid:md11: device dm-101 operational as raid disk 6 [57820.116844] md/raid:md11: device dm-89 operational as raid disk 5 [57820.122946] md/raid:md11: device dm-88 operational as raid disk 4 [57820.129047] md/raid:md11: device dm-76 operational as raid disk 3 [57820.135148] md/raid:md11: device dm-75 operational as raid disk 2 [57820.141248] md/raid:md11: device dm-63 operational as raid disk 1 [57820.148170] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [57820.165124] md11: detected capacity change from 0 to 64011422924800 [57820.181558] md: md5 stopped. [57820.208551] md/raid:md5: device dm-2 operational as raid disk 0 [57820.214490] md/raid:md5: device dm-55 operational as raid disk 9 [57820.220510] md/raid:md5: device dm-54 operational as raid disk 8 [57820.226525] md/raid:md5: device dm-42 operational as raid disk 7 [57820.232551] md/raid:md5: device dm-41 operational as raid disk 6 [57820.238563] md/raid:md5: device dm-29 operational as raid disk 5 [57820.244596] md/raid:md5: device dm-28 operational as raid disk 4 [57820.250619] md/raid:md5: device dm-16 operational as raid disk 3 [57820.256637] md/raid:md5: device dm-15 operational as raid disk 2 [57820.262683] md/raid:md5: device dm-3 operational as raid disk 1 [57820.271103] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [57820.289971] md5: detected capacity change from 0 to 64011422924800 [57820.545299] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [57820.834266] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [57820.883769] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57820.887440] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [57821.162068] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57821.197529] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57821.378320] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [57821.595311] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [57821.612357] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [57821.734551] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57821.930294] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57821.942906] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [57823.469173] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [57826.469242] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [57827.469261] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 4 seconds [57829.469312] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 6 seconds [57829.479305] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [57829.490523] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [57829.500636] Lustre: fir-OST0021: in recovery but waiting for the first client to connect [57832.491623] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [57832.508759] LustreError: Skipped 4 previous similar messages [57833.469415] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 10 seconds [57833.479496] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [57833.772747] Lustre: fir-OST0021: Will be in recovery for at least 5:00, or until 5 clients reconnect [57833.782153] Lustre: fir-OST0021: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [57834.313091] Lustre: fir-OST0021: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [57834.323536] Lustre: Skipped 1 previous similar message [57834.471504] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [57834.480229] Lustre: fir-OST0019: in recovery but waiting for the first client to connect [57837.970020] Lustre: fir-OST0019: Will be in recovery for at least 5:00, or until 5 clients reconnect [57837.979198] Lustre: fir-OST0019: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [57837.979294] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [57839.471562] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [57839.483451] Lustre: fir-OST001f: in recovery but waiting for the first client to connect [57840.157931] Lustre: fir-OST0019: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [57840.158046] Lustre: fir-OST001f: Will be in recovery for at least 5:00, or until 5 clients reconnect [57840.177518] Lustre: Skipped 1 previous similar message [57840.634760] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.9.101.59@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [57840.652128] LustreError: Skipped 3 previous similar messages [57841.469605] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 18 seconds [57841.479690] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [57842.741186] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [57842.758562] LustreError: Skipped 4 previous similar messages [57845.471737] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [57845.515289] Lustre: fir-OST0023: in recovery but waiting for the first client to connect [57847.592279] Lustre: fir-OST0023: Will be in recovery for at least 5:00, or until 5 clients reconnect [57847.601455] Lustre: fir-OST0023: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [57847.611647] Lustre: Skipped 6 previous similar messages [57850.472025] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [57850.480842] Lustre: fir-OST001d: in recovery but waiting for the first client to connect [57855.498597] Lustre: fir-OST001b: Will be in recovery for at least 5:00, or until 5 clients reconnect [57855.507730] Lustre: Skipped 1 previous similar message [57857.492405] Lustre: fir-OST0019: Connection restored to (at 10.9.0.2@o2ib4) [57857.499466] Lustre: Skipped 13 previous similar messages [57857.539448] Lustre: fir-OST0023: Recovery over after 0:10, of 5 clients 5 recovered and 0 were evicted. [57857.548838] Lustre: Skipped 2 previous similar messages [57858.040355] Lustre: fir-OST001b: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 5 known clients (2 recovered, 0 in progress, and 0 evicted) already passed deadline 0:02 [57858.992674] Lustre: fir-OST001b: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 5 known clients (3 recovered, 0 in progress, and 0 evicted) already passed deadline 0:03 [57859.011602] Lustre: Skipped 3 previous similar messages [57859.697527] Lustre: fir-OST001b: Recovery over after 0:04, of 5 clients 5 recovered and 0 were evicted. [57859.706930] Lustre: Skipped 2 previous similar messages [57883.224546] Lustre: fir-OST001b: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [57883.233770] Lustre: Skipped 23 previous similar messages [57909.445880] Lustre: fir-OST0023: deleting orphan objects from 0x0:3 to 0x0:33 [57910.203083] Lustre: fir-OST001f: deleting orphan objects from 0xe00000402:24 to 0xe00000402:65 [57910.211117] Lustre: fir-OST0023: deleting orphan objects from 0x980000402:23 to 0x980000402:65 [57910.214556] Lustre: fir-OST0021: deleting orphan objects from 0x780000402:23 to 0x780000402:65 [57910.220883] Lustre: fir-OST001d: deleting orphan objects from 0x600000402:23 to 0x600000402:65 [57910.231429] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:23 to 0xd00000402:65 [57910.274763] Lustre: fir-OST0019: deleting orphan objects from 0xdc0000402:23 to 0xdc0000402:65 [58732.574093] Lustre: fir-OST001d: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [58732.584355] Lustre: Skipped 7 previous similar messages [58841.213247] Lustre: fir-OST0019: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [58841.223513] Lustre: Skipped 6 previous similar messages [58991.786478] Lustre: fir-OST001f: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [58992.005608] Lustre: 4730:0:(osd_handler.c:1870:osd_trans_start()) fir-OST001f: credits 8227 > trans_max 8192 [58992.015441] Lustre: 4730:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [58992.024745] Lustre: 4730:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [58992.034525] Lustre: 4730:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [58992.044953] Lustre: 4730:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [58992.054195] Lustre: 4730:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [58992.063605] Pid: 4730, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [58992.073793] Call Trace: [58992.076259] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [58992.082825] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [58992.089849] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [58992.096847] [] ofd_trans_start+0x75/0xf0 [ofd] [58992.102989] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [58992.109639] [] ofd_commitrw+0x4b2/0xa10 [ofd] [58992.115696] [] obd_commitrw+0x9c/0x370 [ptlrpc] [58992.121955] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [58992.128572] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [58992.135513] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [58992.143239] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [58992.149570] [] kthread+0xd1/0xe0 [58992.154496] [] ret_from_fork_nospec_begin+0xe/0x21 [58992.160974] [] 0xffffffffffffffff [58992.166096] Lustre: 4730:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001f: opcode 7: before 8192 < left 8209, rollback = 7 [58992.534176] Lustre: 4733:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [58992.543521] Lustre: 4733:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 22 previous similar messages [58992.553262] Lustre: 4733:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [58992.563035] Lustre: 4733:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 22 previous similar messages [58992.572797] Lustre: 4733:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [58992.583252] Lustre: 4733:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 22 previous similar messages [58992.592993] Lustre: 4733:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [58992.602233] Lustre: 4733:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 22 previous similar messages [58992.611973] Lustre: 4733:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [58992.621384] Lustre: 4733:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 22 previous similar messages [58992.667435] Lustre: 18875:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001f: opcode 7: before 8186 < left 8203, rollback = 7 [58992.679199] Lustre: 18875:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 25 previous similar messages [59018.201002] Lustre: fir-OST0021: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [59018.366622] Lustre: 18875:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0021: opcode 7: before 8192 < left 8209, rollback = 7 [59018.378357] Lustre: 18875:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 33 previous similar messages [59018.388013] Lustre: 18875:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59018.397407] Lustre: 18875:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 37 previous similar messages [59018.407250] Lustre: 18875:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59018.417082] Lustre: 18875:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 37 previous similar messages [59018.426923] Lustre: 18875:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59018.437446] Lustre: 18875:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 37 previous similar messages [59018.447290] Lustre: 18875:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59018.456591] Lustre: 18875:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 37 previous similar messages [59018.466428] Lustre: 18875:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59018.475937] Lustre: 18875:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 37 previous similar messages [59059.164180] Lustre: fir-OST001d: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [59059.366176] Lustre: 4730:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001d: opcode 7: before 8192 < left 8209, rollback = 7 [59059.377827] Lustre: 4730:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 59 previous similar messages [59059.387420] Lustre: 4730:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59059.396725] Lustre: 4730:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 59 previous similar messages [59059.406499] Lustre: 4730:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59059.416249] Lustre: 4730:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 59 previous similar messages [59059.426015] Lustre: 4730:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59059.436444] Lustre: 4730:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 59 previous similar messages [59059.446190] Lustre: 4730:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59059.455407] Lustre: 4730:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 59 previous similar messages [59059.465155] Lustre: 4730:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59059.474541] Lustre: 4730:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 59 previous similar messages [59063.411884] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001d: opcode 7: before 8192 < left 8209, rollback = 7 [59063.423620] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 131 previous similar messages [59063.433364] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59063.442760] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 131 previous similar messages [59063.452688] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59063.462519] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 131 previous similar messages [59063.472449] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59063.482972] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 131 previous similar messages [59063.492903] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59063.502211] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 131 previous similar messages [59063.512143] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59063.521624] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 131 previous similar messages [59071.432747] Lustre: 4733:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001d: opcode 7: before 8192 < left 8209, rollback = 7 [59071.444419] Lustre: 4733:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 295 previous similar messages [59071.454076] Lustre: 4733:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59071.463381] Lustre: 4733:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 295 previous similar messages [59071.473238] Lustre: 4733:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59071.482976] Lustre: 4733:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 295 previous similar messages [59071.492803] Lustre: 4733:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59071.503251] Lustre: 4733:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 295 previous similar messages [59071.513074] Lustre: 4733:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59071.522297] Lustre: 4733:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 295 previous similar messages [59071.532154] Lustre: 4733:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59071.541544] Lustre: 4733:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 295 previous similar messages [59091.363144] Lustre: 19823:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0023: opcode 7: before 8192 < left 8209, rollback = 7 [59091.374887] Lustre: 19823:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 561 previous similar messages [59091.384628] Lustre: 19823:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59091.394021] Lustre: 19823:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 561 previous similar messages [59091.403953] Lustre: 19823:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59091.413800] Lustre: 19823:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 561 previous similar messages [59091.423734] Lustre: 19823:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59091.434269] Lustre: 19823:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 561 previous similar messages [59091.444185] Lustre: 19823:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59091.453508] Lustre: 19823:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 561 previous similar messages [59091.463433] Lustre: 19823:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59091.472928] Lustre: 19823:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 561 previous similar messages [59123.367216] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0023: opcode 7: before 8192 < left 8209, rollback = 7 [59123.378964] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 571 previous similar messages [59123.388747] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59123.398146] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 571 previous similar messages [59123.408090] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59123.417950] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 571 previous similar messages [59123.427891] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59123.438436] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 571 previous similar messages [59123.448383] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59123.457693] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 571 previous similar messages [59123.467642] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59123.477127] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 571 previous similar messages [59123.913743] Lustre: fir-OST001d: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [59123.924065] Lustre: Skipped 4 previous similar messages [59187.479353] Lustre: 4730:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0023: opcode 7: before 8192 < left 8209, rollback = 7 [59187.491000] Lustre: 4730:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1121 previous similar messages [59187.500748] Lustre: 4730:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59187.510082] Lustre: 4730:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1121 previous similar messages [59187.520010] Lustre: 4730:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59187.529782] Lustre: 4730:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1121 previous similar messages [59187.539690] Lustre: 4730:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59187.550141] Lustre: 4730:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1121 previous similar messages [59187.560050] Lustre: 4730:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59187.569289] Lustre: 4730:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1121 previous similar messages [59187.579218] Lustre: 4730:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59187.588609] Lustre: 4730:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1121 previous similar messages [59266.820158] Lustre: 18878:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0023: credits 8231 > trans_max 8192 [59266.830084] Pid: 18878, comm: ll_ost_io00_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59266.840370] Call Trace: [59266.842840] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59266.849443] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59266.856493] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59266.863489] [] ofd_trans_start+0x75/0xf0 [ofd] [59266.869634] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59266.876281] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59266.882336] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59266.888615] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59266.895226] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59266.902169] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59266.909899] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59266.916228] [] kthread+0xd1/0xe0 [59266.921140] [] ret_from_fork_nospec_begin+0xe/0x21 [59266.927631] [] 0xffffffffffffffff [59301.902062] Lustre: fir-OST0023: Connection restored to 5b7691fe-133d-27a3-dc18-0b1823004f05 (at 10.0.10.3@o2ib7) [59301.912350] Lustre: Skipped 1 previous similar message [59315.610188] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001b: opcode 7: before 8192 < left 8209, rollback = 7 [59315.621946] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1925 previous similar messages [59315.631804] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59315.637160] Lustre: 18877:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59315.637163] Lustre: 18877:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1926 previous similar messages [59315.637165] Lustre: 18877:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59315.637167] Lustre: 18877:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1926 previous similar messages [59315.637169] Lustre: 18877:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59315.637170] Lustre: 18877:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1926 previous similar messages [59315.637172] Lustre: 18877:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59315.637174] Lustre: 18877:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1926 previous similar messages [59315.720366] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1928 previous similar messages [59326.968744] Lustre: 18877:0:(osd_handler.c:1870:osd_trans_start()) fir-OST001b: credits 8227 > trans_max 8192 [59326.978664] Pid: 18877, comm: ll_ost_io01_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59326.988923] Call Trace: [59326.991394] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59326.997968] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59327.004960] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59327.011964] [] ofd_trans_start+0x75/0xf0 [ofd] [59327.018111] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59327.024764] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59327.030822] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59327.037097] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59327.043708] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59327.050654] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59327.058383] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59327.064712] [] kthread+0xd1/0xe0 [59327.069624] [] ret_from_fork_nospec_begin+0xe/0x21 [59327.076115] [] 0xffffffffffffffff [59462.470243] Lustre: 18877:0:(osd_handler.c:1870:osd_trans_start()) fir-OST001f: credits 8215 > trans_max 8192 [59462.480180] Pid: 18877, comm: ll_ost_io01_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59462.490456] Call Trace: [59462.492926] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59462.499498] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59462.506526] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59462.513531] [] ofd_trans_start+0x75/0xf0 [ofd] [59462.519702] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59462.526348] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59462.532404] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59462.538681] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59462.545274] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59462.552229] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59462.559948] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59462.566279] [] kthread+0xd1/0xe0 [59462.571212] [] ret_from_fork_nospec_begin+0xe/0x21 [59462.577689] [] 0xffffffffffffffff [59522.597827] Lustre: 4730:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0021: credits 8227 > trans_max 8192 [59522.607659] Pid: 4730, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59522.617858] Call Trace: [59522.620327] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59522.626901] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59522.633918] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59522.640930] [] ofd_trans_start+0x75/0xf0 [ofd] [59522.647073] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59522.653723] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59522.659779] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59522.666068] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59522.672681] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59522.679628] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59522.687357] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59522.693684] [] kthread+0xd1/0xe0 [59522.698614] [] ret_from_fork_nospec_begin+0xe/0x21 [59522.705092] [] 0xffffffffffffffff [59571.721582] Lustre: 18876:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0021: opcode 7: before 8192 < left 8209, rollback = 7 [59571.733317] Lustre: 18876:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 2681 previous similar messages [59571.743148] Lustre: 18876:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [59571.752541] Lustre: 18876:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 2682 previous similar messages [59571.762574] Lustre: 18876:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [59571.772395] Lustre: 18876:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 2684 previous similar messages [59571.782414] Lustre: 18876:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [59571.792952] Lustre: 18876:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 2684 previous similar messages [59571.802948] Lustre: 18876:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [59571.812269] Lustre: 18876:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 2684 previous similar messages [59571.822279] Lustre: 18876:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [59571.831756] Lustre: 18876:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 2684 previous similar messages [59584.030264] Lustre: 18876:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0021: credits 8231 > trans_max 8192 [59584.040211] Pid: 18876, comm: ll_ost_io00_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [59584.050485] Call Trace: [59584.052947] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [59584.059523] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [59584.066548] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [59584.073565] [] ofd_trans_start+0x75/0xf0 [ofd] [59584.079708] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [59584.086371] [] ofd_commitrw+0x4b2/0xa10 [ofd] [59584.092417] [] obd_commitrw+0x9c/0x370 [ptlrpc] [59584.098688] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [59584.105278] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [59584.112224] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [59584.119940] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [59584.126284] [] kthread+0xd1/0xe0 [59584.131199] [] ret_from_fork_nospec_begin+0xe/0x21 [59584.137667] [] 0xffffffffffffffff [59816.878630] Lustre: fir-OST001d: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [59816.888898] Lustre: Skipped 12 previous similar messages [61368.525389] Lustre: fir-OST0019: haven't heard from client e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) in 228 seconds. I think it's dead, and I am evicting it. exp ffff9284592da400, cur 1547839068 expire 1547838918 last 1547838840 [64115.314198] Lustre: fir-OST0019: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [64115.324647] Lustre: Skipped 11 previous similar messages [64186.117113] Lustre: fir-OST0021: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [64186.127547] Lustre: Skipped 5 previous similar messages [64198.371360] Lustre: 4736:0:(osd_handler.c:1870:osd_trans_start()) fir-OST001d: credits 8227 > trans_max 8192 [64198.381214] Lustre: 4736:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64198.390519] Lustre: 4736:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 654 previous similar messages [64198.400367] Lustre: 4736:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64198.410101] Lustre: 4736:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 654 previous similar messages [64198.419925] Lustre: 4736:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64198.430373] Lustre: 4736:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 654 previous similar messages [64198.440194] Lustre: 4736:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64198.449418] Lustre: 4736:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 654 previous similar messages [64198.459264] Lustre: 4736:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64198.468655] Lustre: 4736:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 654 previous similar messages [64198.478495] Pid: 4736, comm: ll_ost_io03_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64198.488667] Call Trace: [64198.491132] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64198.497708] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64198.504726] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64198.511729] [] ofd_trans_start+0x75/0xf0 [ofd] [64198.517869] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64198.524521] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64198.530577] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64198.536846] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64198.543468] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64198.550409] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64198.558138] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64198.564467] [] kthread+0xd1/0xe0 [64198.569395] [] ret_from_fork_nospec_begin+0xe/0x21 [64198.575874] [] 0xffffffffffffffff [64198.580969] Lustre: 4736:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001d: opcode 7: before 8192 < left 8209, rollback = 7 [64198.592613] Lustre: 4736:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 653 previous similar messages [64262.442428] Lustre: 18875:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64262.451832] Lustre: 18875:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1458 previous similar messages [64262.461849] Lustre: 18875:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64262.471700] Lustre: 18875:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1458 previous similar messages [64262.481697] Lustre: 18875:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64262.492263] Lustre: 18875:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1458 previous similar messages [64262.502265] Lustre: 18875:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64262.511587] Lustre: 18875:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1458 previous similar messages [64262.521605] Lustre: 18875:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64262.531118] Lustre: 18875:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1458 previous similar messages [64262.675947] Lustre: 81019:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0023: opcode 7: before 8192 < left 8209, rollback = 7 [64262.687686] Lustre: 81019:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1461 previous similar messages [64390.487548] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64390.496941] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1949 previous similar messages [64390.506962] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64390.516807] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1949 previous similar messages [64390.526829] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64390.537388] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1949 previous similar messages [64390.547391] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64390.556700] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1949 previous similar messages [64390.566718] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64390.576200] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1949 previous similar messages [64390.753951] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0023: opcode 7: before 8192 < left 8209, rollback = 7 [64390.765682] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1949 previous similar messages [64403.614410] Lustre: 18878:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0023: credits 8231 > trans_max 8192 [64403.624333] Pid: 18878, comm: ll_ost_io00_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64403.634601] Call Trace: [64403.637071] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64403.643669] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64403.650678] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64403.657688] [] ofd_trans_start+0x75/0xf0 [ofd] [64403.663817] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64403.670481] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64403.676522] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64403.682803] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64403.689394] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64403.696355] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64403.704068] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64403.710406] [] kthread+0xd1/0xe0 [64403.715323] [] ret_from_fork_nospec_begin+0xe/0x21 [64403.721814] [] 0xffffffffffffffff [64463.981557] Lustre: 81019:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0023: credits 8227 > trans_max 8192 [64463.991487] Pid: 81019, comm: ll_ost_io03_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64464.001785] Call Trace: [64464.004254] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64464.010829] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64464.017843] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64464.024850] [] ofd_trans_start+0x75/0xf0 [ofd] [64464.030999] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64464.037651] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64464.043713] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64464.049985] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64464.056606] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64464.063556] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64464.071292] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64464.077629] [] kthread+0xd1/0xe0 [64464.082558] [] ret_from_fork_nospec_begin+0xe/0x21 [64464.089037] [] 0xffffffffffffffff [64529.131357] Lustre: 18878:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0019: credits 8231 > trans_max 8192 [64529.141279] Pid: 18878, comm: ll_ost_io00_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64529.151559] Call Trace: [64529.154032] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64529.160645] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64529.167660] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64529.174655] [] ofd_trans_start+0x75/0xf0 [ofd] [64529.180822] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64529.187487] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64529.193544] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64529.199822] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64529.206435] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64529.213379] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64529.221108] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64529.227436] [] kthread+0xd1/0xe0 [64529.232363] [] ret_from_fork_nospec_begin+0xe/0x21 [64529.238842] [] 0xffffffffffffffff [64589.411275] Lustre: 4736:0:(osd_handler.c:1870:osd_trans_start()) fir-OST001b: credits 8227 > trans_max 8192 [64589.421111] Pid: 4736, comm: ll_ost_io03_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64589.431286] Call Trace: [64589.433762] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64589.440332] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64589.447343] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64589.454348] [] ofd_trans_start+0x75/0xf0 [ofd] [64589.460491] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64589.467138] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64589.473195] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64589.479465] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64589.486066] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64589.493008] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64589.500737] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64589.507067] [] kthread+0xd1/0xe0 [64589.512000] [] ret_from_fork_nospec_begin+0xe/0x21 [64589.518476] [] 0xffffffffffffffff [64589.523541] Pid: 4730, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64589.533711] Call Trace: [64589.536176] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64589.542749] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64589.549761] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64589.556762] [] ofd_trans_start+0x75/0xf0 [ofd] [64589.562906] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64589.569556] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64589.575609] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64589.581871] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64589.588485] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64589.595417] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64589.603145] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64589.609474] [] kthread+0xd1/0xe0 [64589.614402] [] ret_from_fork_nospec_begin+0xe/0x21 [64589.620881] [] 0xffffffffffffffff [64646.525999] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [64646.535395] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 3672 previous similar messages [64646.545426] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [64646.555259] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 3672 previous similar messages [64646.565279] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [64646.575798] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 3672 previous similar messages [64646.585817] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [64646.595122] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 3672 previous similar messages [64646.605144] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [64646.614627] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 3672 previous similar messages [64646.787366] Lustre: 4736:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0019: opcode 7: before 8192 < left 8209, rollback = 7 [64646.799014] Lustre: 4736:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 3667 previous similar messages [64781.968650] Lustre: 18877:0:(osd_handler.c:1870:osd_trans_start()) fir-OST001b: credits 8231 > trans_max 8192 [64781.978569] Lustre: 18877:0:(osd_handler.c:1870:osd_trans_start()) Skipped 1 previous similar message [64781.987798] Pid: 18877, comm: ll_ost_io01_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64781.998078] Call Trace: [64782.000547] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64782.007126] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64782.014138] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64782.021146] [] ofd_trans_start+0x75/0xf0 [ofd] [64782.027289] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64782.033946] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64782.040013] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64782.046317] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64782.052918] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64782.059883] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64782.067597] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64782.073940] [] kthread+0xd1/0xe0 [64782.078856] [] ret_from_fork_nospec_begin+0xe/0x21 [64782.085345] [] 0xffffffffffffffff [64842.164859] Pid: 18878, comm: ll_ost_io00_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64842.175116] Call Trace: [64842.177605] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64842.184178] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64842.191191] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64842.198193] [] ofd_trans_start+0x75/0xf0 [ofd] [64842.204354] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64842.211004] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64842.217045] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64842.223311] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64842.229906] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64842.236849] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64842.244572] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64842.250898] [] kthread+0xd1/0xe0 [64842.255811] [] ret_from_fork_nospec_begin+0xe/0x21 [64842.262285] [] 0xffffffffffffffff [64842.267315] Pid: 18877, comm: ll_ost_io01_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64842.277590] Call Trace: [64842.280075] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64842.286649] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64842.293644] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64842.300677] [] ofd_trans_start+0x75/0xf0 [ofd] [64842.306823] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64842.313490] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64842.319545] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64842.325834] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64842.332440] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64842.339388] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64842.347129] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64842.353461] [] kthread+0xd1/0xe0 [64842.358390] [] ret_from_fork_nospec_begin+0xe/0x21 [64842.364867] [] 0xffffffffffffffff [64973.185640] Lustre: 19823:0:(osd_handler.c:1870:osd_trans_start()) fir-OST001b: credits 8231 > trans_max 8192 [64973.195557] Lustre: 19823:0:(osd_handler.c:1870:osd_trans_start()) Skipped 2 previous similar messages [64973.204897] Pid: 19823, comm: ll_ost_io02_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64973.215178] Call Trace: [64973.217649] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64973.224221] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64973.231242] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64973.238250] [] ofd_trans_start+0x75/0xf0 [ofd] [64973.244426] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64973.251075] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64973.257161] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64973.263435] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64973.270054] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64973.277017] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64973.284756] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64973.291109] [] kthread+0xd1/0xe0 [64973.296044] [] ret_from_fork_nospec_begin+0xe/0x21 [64973.302522] [] 0xffffffffffffffff [64973.307589] Pid: 85541, comm: ll_ost_io03_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [64973.317853] Call Trace: [64973.320327] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [64973.326918] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [64973.333929] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [64973.340974] [] ofd_trans_start+0x75/0xf0 [ofd] [64973.347124] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [64973.353792] [] ofd_commitrw+0x4b2/0xa10 [ofd] [64973.359847] [] obd_commitrw+0x9c/0x370 [ptlrpc] [64973.366126] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [64973.372752] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [64973.379697] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [64973.387427] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [64973.393756] [] kthread+0xd1/0xe0 [64973.398682] [] ret_from_fork_nospec_begin+0xe/0x21 [64973.405160] [] 0xffffffffffffffff [65033.427554] Pid: 18876, comm: ll_ost_io00_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65033.437833] Call Trace: [65033.440299] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65033.446871] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65033.453898] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65033.460902] [] ofd_trans_start+0x75/0xf0 [ofd] [65033.467052] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65033.473704] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65033.479758] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65033.486030] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65033.492644] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65033.499583] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65033.507312] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65033.513640] [] kthread+0xd1/0xe0 [65033.518555] [] ret_from_fork_nospec_begin+0xe/0x21 [65033.525036] [] 0xffffffffffffffff [65093.832795] Pid: 18876, comm: ll_ost_io00_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65093.843058] Call Trace: [65093.845522] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65093.852095] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65093.859106] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65093.866111] [] ofd_trans_start+0x75/0xf0 [ofd] [65093.872261] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65093.878916] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65093.884986] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65093.891262] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65093.897891] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65093.904853] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65093.912588] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65093.918934] [] kthread+0xd1/0xe0 [65093.923871] [] ret_from_fork_nospec_begin+0xe/0x21 [65093.930385] [] 0xffffffffffffffff [65153.941160] Lustre: 85541:0:(osd_handler.c:1870:osd_trans_start()) fir-OST0019: credits 8227 > trans_max 8192 [65153.951146] Lustre: 85541:0:(osd_handler.c:1870:osd_trans_start()) Skipped 3 previous similar messages [65153.960486] Pid: 85541, comm: ll_ost_io03_004 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65153.970762] Call Trace: [65153.973233] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65153.979824] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65153.986836] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65153.993842] [] ofd_trans_start+0x75/0xf0 [ofd] [65153.999985] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65154.006635] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65154.012698] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65154.018967] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65154.025576] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65154.032521] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65154.040258] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65154.046588] [] kthread+0xd1/0xe0 [65154.051514] [] ret_from_fork_nospec_begin+0xe/0x21 [65154.057995] [] 0xffffffffffffffff [65158.653541] Lustre: 4733:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [65158.662852] Lustre: 4733:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 7343 previous similar messages [65158.672773] Lustre: 4733:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [65158.682512] Lustre: 4733:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 7343 previous similar messages [65158.692444] Lustre: 4733:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [65158.693038] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [65158.693042] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 7343 previous similar messages [65158.693044] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [65158.693046] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 7343 previous similar messages [65158.741679] Lustre: 4733:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 7345 previous similar messages [65158.927741] Lustre: 85541:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001b: opcode 7: before 8192 < left 8209, rollback = 7 [65158.939473] Lustre: 85541:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 7335 previous similar messages [65260.278592] Pid: 4730, comm: ll_ost_io01_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65260.288764] Call Trace: [65260.291259] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65260.297827] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65260.304853] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65260.311860] [] ofd_trans_start+0x75/0xf0 [ofd] [65260.318013] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65260.324661] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65260.330715] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65260.336998] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65260.343616] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65260.350573] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65260.358298] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65260.364656] [] kthread+0xd1/0xe0 [65260.369572] [] ret_from_fork_nospec_begin+0xe/0x21 [65260.376087] [] 0xffffffffffffffff [65320.480734] Pid: 18876, comm: ll_ost_io00_003 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65320.490995] Call Trace: [65320.493460] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65320.500033] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65320.507059] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65320.514068] [] ofd_trans_start+0x75/0xf0 [ofd] [65320.520216] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65320.526866] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65320.532921] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65320.539193] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65320.545809] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65320.552756] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65320.560482] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65320.566811] [] kthread+0xd1/0xe0 [65320.571742] [] ret_from_fork_nospec_begin+0xe/0x21 [65320.578217] [] 0xffffffffffffffff [65320.583249] Pid: 4736, comm: ll_ost_io03_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [65320.593422] Call Trace: [65320.595896] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [65320.602468] [] libcfs_debug_dumpstack+0x26/0x30 [libcfs] [65320.609535] [] osd_trans_start+0x4b6/0x4e0 [osd_ldiskfs] [65320.616544] [] ofd_trans_start+0x75/0xf0 [ofd] [65320.622699] [] ofd_commitrw_write+0xa31/0x1d40 [ofd] [65320.629352] [] ofd_commitrw+0x4b2/0xa10 [ofd] [65320.635408] [] obd_commitrw+0x9c/0x370 [ptlrpc] [65320.641678] [] tgt_brw_write+0x100d/0x1a90 [ptlrpc] [65320.648278] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [65320.655214] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [65320.662943] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [65320.669271] [] kthread+0xd1/0xe0 [65320.674185] [] ret_from_fork_nospec_begin+0xe/0x21 [65320.680687] [] 0xffffffffffffffff [65758.692786] Lustre: 18896:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [65758.702182] Lustre: 18896:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 8454 previous similar messages [65758.712193] Lustre: 18896:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [65758.722025] Lustre: 18896:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 8454 previous similar messages [65758.732041] Lustre: 18896:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [65758.742576] Lustre: 18896:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 8452 previous similar messages [65758.752569] Lustre: 18896:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [65758.761875] Lustre: 18896:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 8454 previous similar messages [65758.771891] Lustre: 18896:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [65758.781370] Lustre: 18896:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 8454 previous similar messages [65758.952319] Lustre: 85541:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001d: opcode 7: before 8192 < left 8209, rollback = 7 [65758.964073] Lustre: 85541:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 8455 previous similar messages [73965.799397] Lustre: fir-OST0021: Connection restored to e2111f28-9326-139c-a941-4106e4359611 (at 10.9.101.59@o2ib4) [73965.809843] Lustre: Skipped 2 previous similar messages [81560.881122] Lustre: fir-OST0021: Connection restored to 77d349ca-d6a8-608c-17aa-a92d625d5ed3 (at 10.8.6.33@o2ib6) [81560.891400] Lustre: Skipped 5 previous similar messages [81573.239142] Lustre: 18875:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST001b: opcode 7: before 8192 < left 8209, rollback = 7 [81573.250873] Lustre: 18875:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 809 previous similar messages [81573.260622] Lustre: 18875:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [81573.270020] Lustre: 18875:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 817 previous similar messages [81573.279951] Lustre: 18875:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [81573.289779] Lustre: 18875:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 817 previous similar messages [81573.299711] Lustre: 18875:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [81573.310231] Lustre: 18875:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 817 previous similar messages [81573.320154] Lustre: 18875:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [81573.329456] Lustre: 18875:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 817 previous similar messages [81573.339387] Lustre: 18875:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [81573.348860] Lustre: 18875:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 817 previous similar messages [81648.331649] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0019: opcode 7: before 8192 < left 8209, rollback = 7 [81648.343380] Lustre: 18878:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 1757 previous similar messages [81648.353232] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [81648.362629] Lustre: 18878:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 1757 previous similar messages [81648.372680] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [81648.382502] Lustre: 18878:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 1757 previous similar messages [81648.392521] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [81648.403041] Lustre: 18878:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 1757 previous similar messages [81648.413059] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [81648.422368] Lustre: 18878:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 1757 previous similar messages [81648.432366] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [81648.441846] Lustre: 18878:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 1757 previous similar messages [81798.362427] Lustre: 18877:0:(osd_internal.h:1205:osd_trans_exec_op()) fir-OST0019: opcode 7: before 8192 < left 8209, rollback = 7 [81798.374160] Lustre: 18877:0:(osd_internal.h:1205:osd_trans_exec_op()) Skipped 2469 previous similar messages [81798.384021] Lustre: 18877:0:(osd_handler.c:1799:osd_trans_dump_creds()) create: 0/0/0, destroy: 0/0/0 [81798.393426] Lustre: 18877:0:(osd_handler.c:1799:osd_trans_dump_creds()) Skipped 2469 previous similar messages [81798.403449] Lustre: 18877:0:(osd_handler.c:1806:osd_trans_dump_creds()) attr_set: 1/1/0, xattr_set: 2/15/0 [81798.413274] Lustre: 18877:0:(osd_handler.c:1806:osd_trans_dump_creds()) Skipped 2469 previous similar messages [81798.423297] Lustre: 18877:0:(osd_handler.c:1816:osd_trans_dump_creds()) write: 2/8209/0, punch: 0/0/0, quota 2/2/0 [81798.433816] Lustre: 18877:0:(osd_handler.c:1816:osd_trans_dump_creds()) Skipped 2469 previous similar messages [81798.443834] Lustre: 18877:0:(osd_handler.c:1823:osd_trans_dump_creds()) insert: 0/0/0, delete: 0/0/0 [81798.453142] Lustre: 18877:0:(osd_handler.c:1823:osd_trans_dump_creds()) Skipped 2469 previous similar messages [81798.463163] Lustre: 18877:0:(osd_handler.c:1830:osd_trans_dump_creds()) ref_add: 0/0/0, ref_del: 0/0/0 [81798.472646] Lustre: 18877:0:(osd_handler.c:1830:osd_trans_dump_creds()) Skipped 2471 previous similar messages [81897.314038] Lustre: 2437:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547859589/real 1547859589] req@ffff9293db010600 x1623023171267648/t0(0) o400->fir-MDT0003-lwp-OST001d@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547859596 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [81897.314043] Lustre: fir-MDT0001-lwp-OST001b: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [81897.314044] Lustre: fir-MDT0000-lwp-OST001d: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [81897.314045] Lustre: Skipped 1 previous similar message [81897.314046] Lustre: Skipped 1 previous similar message [81897.384557] Lustre: 2437:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 15 previous similar messages [81899.025085] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [81899.035082] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [81899.583089] Lustre: 29389:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1547859593/real 1547859593] req@ffff9293dc26aa00 x1623023171267776/t0(0) o39->fir-MDT0002-lwp-OST001b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1547859599 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [81899.590388] Lustre: Failing over fir-OST0023 [81899.615654] Lustre: 29389:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [81900.362649] Lustre: server umount fir-OST001f complete [81901.025161] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [81901.035159] LNet: 2312:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 10 previous similar messages [81901.048165] Lustre: Failing over fir-OST0021 [81901.052444] Lustre: Skipped 4 previous similar messages [81901.376306] Lustre: server umount fir-OST0019 complete [81901.381449] Lustre: Skipped 2 previous similar messages [81903.717821] md5: detected capacity change from 64011422924800 to 0 [81903.724022] md: md5 stopped. [81903.739100] md7: detected capacity change from 64011422924800 to 0 [81903.745290] md: md7 stopped. [81904.140023] Lustre: server umount fir-OST0021 complete [81904.145195] Lustre: Skipped 1 previous similar message [81904.392307] md11: detected capacity change from 64011422924800 to 0 [81904.398590] md: md11 stopped. [81909.356866] md9: detected capacity change from 64011422924800 to 0 [81909.363067] md: md9 stopped. [81909.385628] md3: detected capacity change from 64011422924800 to 0 [81909.391822] md: md3 stopped. [81910.824501] md1: detected capacity change from 64011422924800 to 0 [81910.830695] md: md1 stopped. [81911.592137] md: md3 stopped. [81913.613027] LNetError: 850:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.201@o2ib7 on NA (ib0:1:10.0.10.106): bad dst nid 10.0.10.106@o2ib7 [81915.065440] LNet: Removed LNI 10.0.10.106@o2ib7 [81963.389930] md: md11 stopped. [81963.446577] md/raid:md11: device dm-62 operational as raid disk 0 [81963.452683] md/raid:md11: device dm-115 operational as raid disk 9 [81963.458874] md/raid:md11: device dm-114 operational as raid disk 8 [81963.465065] md/raid:md11: device dm-102 operational as raid disk 7 [81963.471258] md/raid:md11: device dm-101 operational as raid disk 6 [81963.477472] md/raid:md11: device dm-89 operational as raid disk 5 [81963.483572] md/raid:md11: device dm-88 operational as raid disk 4 [81963.489700] md/raid:md11: device dm-76 operational as raid disk 3 [81963.495804] md/raid:md11: device dm-75 operational as raid disk 2 [81963.501905] md/raid:md11: device dm-63 operational as raid disk 1 [81963.508862] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [81963.527539] md11: detected capacity change from 0 to 64011422924800 [81963.544415] md: md1 stopped. [81963.564886] md/raid:md1: device dm-12 operational as raid disk 0 [81963.570898] md/raid:md1: device dm-47 operational as raid disk 9 [81963.576912] md/raid:md1: device dm-46 operational as raid disk 8 [81963.582924] md/raid:md1: device dm-33 operational as raid disk 7 [81963.588930] md/raid:md1: device dm-32 operational as raid disk 6 [81963.594938] md/raid:md1: device dm-20 operational as raid disk 5 [81963.600953] md/raid:md1: device dm-19 operational as raid disk 4 [81963.606958] md/raid:md1: device dm-7 operational as raid disk 3 [81963.612878] md/raid:md1: device dm-6 operational as raid disk 2 [81963.618798] md/raid:md1: device dm-23 operational as raid disk 1 [81963.625591] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [81963.644019] md1: detected capacity change from 0 to 64011422924800 [81963.661016] md: md5 stopped. [81963.682343] md/raid:md5: device dm-2 operational as raid disk 0 [81963.688281] md/raid:md5: device dm-55 operational as raid disk 9 [81963.694298] md/raid:md5: device dm-54 operational as raid disk 8 [81963.700313] md/raid:md5: device dm-42 operational as raid disk 7 [81963.706327] md/raid:md5: device dm-41 operational as raid disk 6 [81963.712342] md/raid:md5: device dm-29 operational as raid disk 5 [81963.718355] md/raid:md5: device dm-28 operational as raid disk 4 [81963.724371] md/raid:md5: device dm-16 operational as raid disk 3 [81963.730386] md/raid:md5: device dm-15 operational as raid disk 2 [81963.736401] md/raid:md5: device dm-3 operational as raid disk 1 [81963.743119] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [81963.762160] md5: detected capacity change from 0 to 64011422924800 [81963.777514] md: md3 stopped. [81963.848834] md/raid:md3: device dm-56 operational as raid disk 0 [81963.854860] md/raid:md3: device dm-51 operational as raid disk 9 [81963.860877] md/raid:md3: device dm-50 operational as raid disk 8 [81963.866896] md/raid:md3: device dm-38 operational as raid disk 7 [81963.872917] md/raid:md3: device dm-37 operational as raid disk 6 [81963.878930] md/raid:md3: device dm-25 operational as raid disk 5 [81963.884946] md/raid:md3: device dm-24 operational as raid disk 4 [81963.890963] md/raid:md3: device dm-11 operational as raid disk 3 [81963.896979] md/raid:md3: device dm-10 operational as raid disk 2 [81963.903005] md/raid:md3: device dm-57 operational as raid disk 1 [81963.910081] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [81963.929180] md3: detected capacity change from 0 to 64011422924800 [81963.939968] md: md7 stopped. [81964.024277] md/raid:md7: device dm-72 operational as raid disk 0 [81964.030307] md/raid:md7: device dm-107 operational as raid disk 9 [81964.036417] md/raid:md7: device dm-106 operational as raid disk 8 [81964.042530] md/raid:md7: device dm-93 operational as raid disk 7 [81964.048570] md/raid:md7: device dm-92 operational as raid disk 6 [81964.054617] md/raid:md7: device dm-80 operational as raid disk 5 [81964.060645] md/raid:md7: device dm-79 operational as raid disk 4 [81964.066680] md/raid:md7: device dm-67 operational as raid disk 3 [81964.072701] md/raid:md7: device dm-66 operational as raid disk 2 [81964.078735] md/raid:md7: device dm-83 operational as raid disk 1 [81964.090583] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [81964.130892] md7: detected capacity change from 0 to 64011422924800 [81964.146690] md: md9 stopped. [81964.200983] md/raid:md9: device dm-116 operational as raid disk 0 [81964.207139] md/raid:md9: device dm-111 operational as raid disk 9 [81964.213256] md/raid:md9: device dm-110 operational as raid disk 8 [81964.219355] md/raid:md9: device dm-98 operational as raid disk 7 [81964.225382] md/raid:md9: device dm-97 operational as raid disk 6 [81964.231395] md/raid:md9: device dm-85 operational as raid disk 5 [81964.237409] md/raid:md9: device dm-84 operational as raid disk 4 [81964.243423] md/raid:md9: device dm-71 operational as raid disk 3 [81964.249436] md/raid:md9: device dm-70 operational as raid disk 2 [81964.255445] md/raid:md9: device dm-117 operational as raid disk 1 [81964.262234] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [81964.293009] md9: detected capacity change from 0 to 64011422924800 [81982.877168] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [81983.191224] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81984.114189] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [81984.262198] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [81984.423259] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81984.570211] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81984.758420] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [81985.076198] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81985.272226] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [81985.590607] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [81987.137253] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [81987.448578] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [82008.684054] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [82008.691672] alg: No test for adler32 (adler32-zlib) [82009.500900] Lustre: Lustre: Build Version: 2.12.0 [82009.632500] LNet: Using FastReg for registration [82009.648146] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [82038.601961] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [82038.940916] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [82038.945992] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [82039.236384] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [82039.263065] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [82039.413476] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [82039.505764] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [82039.590999] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [82039.713005] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [82039.787542] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [82039.857110] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [82040.037476] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [82040.054474] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [82040.125193] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [82040.202565] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [82040.333488] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [82040.359692] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [82040.367596] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [82040.535324] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [82040.557084] Lustre: fir-OST001d: new disk, initializing [82041.467091] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [82041.562451] Lustre: srv-fir-OST001d: No data found on store. Initialize space [82041.585855] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [82041.639505] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [82041.749522] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [82041.754605] Lustre: fir-OST0023: new disk, initializing [82041.935667] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [82042.052732] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [82042.073894] Lustre: srv-fir-OST0019: No data found on store. Initialize space [82042.081031] Lustre: Skipped 1 previous similar message [82042.094553] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [82042.102735] Lustre: Skipped 1 previous similar message [82042.839058] Lustre: fir-OST0021: new disk, initializing [82042.844305] Lustre: Skipped 2 previous similar messages [82043.130261] Lustre: srv-fir-OST001f: No data found on store. Initialize space [82043.137417] Lustre: Skipped 2 previous similar messages [82043.151968] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [82043.160151] Lustre: Skipped 2 previous similar messages [82043.899420] Lustre: fir-OST0023: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [82045.254632] Lustre: fir-OST001d: Connection restored to fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) [82045.263853] Lustre: Skipped 6 previous similar messages [82045.272460] Lustre: cli-fir-OST001d-super: Allocated super-sequence [0x0000000400000400-0x0000000440000400]:1d:ost] [82046.262090] Lustre: fir-OST0019: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [82046.272270] Lustre: Skipped 2 previous similar messages [82049.817331] Lustre: fir-OST001f: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [82049.827521] Lustre: Skipped 2 previous similar messages [82058.193633] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [82058.210771] LustreError: Skipped 1 previous similar message [82071.666374] Lustre: fir-OST0023: Connection restored to fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) [82071.675595] Lustre: Skipped 6 previous similar messages [82071.705186] Lustre: cli-fir-OST0019-super: Allocated super-sequence [0x00000006c0000400-0x0000000700000400]:19:ost] [82083.194554] Lustre: fir-OST0019: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [82083.204750] Lustre: Skipped 19 previous similar messages [82084.633504] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [82084.650873] LustreError: Skipped 2 previous similar messages [82108.194830] Lustre: fir-OST0021: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [82108.205010] Lustre: Skipped 2 previous similar messages [82181.161357] Lustre: fir-OST001f: Connection restored to ac6fa15d-85d8-d7f7-0237-c13aeb11aa09 (at 10.0.10.3@o2ib7) [82181.161358] Lustre: fir-OST0019: Connection restored to ac6fa15d-85d8-d7f7-0237-c13aeb11aa09 (at 10.0.10.3@o2ib7) [82181.161361] Lustre: Skipped 2 previous similar messages [82181.187090] Lustre: Skipped 4 previous similar messages [82348.805706] Lustre: fir-OST0019: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [82348.816146] Lustre: Skipped 6 previous similar messages [82645.276051] Lustre: fir-OST001b: Connection restored to ac6fa15d-85d8-d7f7-0237-c13aeb11aa09 (at 10.0.10.3@o2ib7) [82645.286317] Lustre: Skipped 3 previous similar messages [92445.987468] Lustre: fir-OST0019: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [92445.997910] Lustre: Skipped 8 previous similar messages [96830.597436] Lustre: fir-OST001b: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [97167.344225] Lustre: fir-OST0021: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [98298.961078] Lustre: fir-OST001f: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [98655.818711] Lustre: fir-OST001d: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [99024.734770] Lustre: fir-OST0023: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [99408.167260] Lustre: fir-OST0019: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [99795.533286] Lustre: fir-OST001b: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [197104.760747] md: data-check of RAID array md9 [197110.760580] md: data-check of RAID array md7 [197116.805018] md: data-check of RAID array md3 [197122.864049] md: data-check of RAID array md5 [197128.963245] md: data-check of RAID array md11 [218632.634117] perf: interrupt took too long (2502 > 2500), lowering kernel.perf_event_max_sample_rate to 79000 [239971.205515] perf: interrupt took too long (3137 > 3127), lowering kernel.perf_event_max_sample_rate to 63000 [291398.597267] md: md5: data-check done. [313317.326241] md: md3: data-check done. [333976.038913] md: md7: data-check done. [402903.337557] Lustre: fir-OST0021: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [402931.585706] Lustre: fir-OST0019: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [402931.596234] Lustre: Skipped 3 previous similar messages [411696.425136] Lustre: fir-OST001d: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [411696.435658] Lustre: Skipped 1 previous similar message [417022.704213] Lustre: fir-OST0019: Connection restored to 703788fd-10ca-aad7-8a77-36cf90d4185c (at 10.9.101.59@o2ib4) [417022.714760] Lustre: Skipped 9 previous similar messages [417101.368939] LNet: 33400:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 7 seconds [417101.379160] Lustre: 33706:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548194792/real 1548194793] req@ffff926426129200 x1623048547574896/t0(0) o400->fir-MDT0003-lwp-OST0021@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194799 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [417101.379171] Lustre: fir-MDT0003-lwp-OST0019: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [417101.424195] Lustre: 33706:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [417102.368980] LNet: 33400:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 21 seconds [417102.379234] LNet: 33400:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [417102.388694] Lustre: 33714:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548194792/real 1548194794] req@ffff92642612a700 x1623048547574816/t0(0) o400->fir-MDT0001-lwp-OST0019@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194799 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [417102.388705] Lustre: fir-MDT0001-lwp-OST0021: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [417102.388707] Lustre: Skipped 6 previous similar messages [417102.438925] Lustre: 33714:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [417103.368998] LNet: 33400:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 5 seconds [417103.379169] LNet: 33400:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [417103.570998] Lustre: 93340:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548194789/real 1548194789] req@ffff9275f43c0000 x1623048547574608/t0(0) o39->fir-MDT0002-lwp-OST0021@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194795 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [417103.571001] Lustre: 93338:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548194789/real 1548194789] req@ffff927423151500 x1623048547574592/t0(0) o39->fir-MDT0002-lwp-OST001f@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194795 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [417103.571005] Lustre: 93338:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [417103.637595] Lustre: 93340:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [417109.572130] Lustre: 93337:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548194795/real 1548194795] req@ffff928d91863600 x1623048547575184/t0(0) o39->fir-MDT0000-lwp-OST0023@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548194801 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [417109.577608] Lustre: Failing over fir-OST001f [417109.577609] Lustre: Failing over fir-OST001b [417109.609250] Lustre: 93337:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [417110.381893] Lustre: server umount fir-OST001b complete [417110.387162] Lustre: Skipped 3 previous similar messages [417110.735254] md7: detected capacity change from 64011422924800 to 0 [417110.741534] md: md7 stopped. [417110.784928] md1: detected capacity change from 64011422924800 to 0 [417110.791206] md: md1 stopped. [417110.833146] md: md9: data-check interrupted. [417110.833909] md5: detected capacity change from 64011422924800 to 0 [417110.833918] md: md5 stopped. [417110.981053] md9: detected capacity change from 64011422924800 to 0 [417110.987332] md: md9 stopped. [417111.073992] md: md11: data-check interrupted. [417111.163113] md3: detected capacity change from 64011422924800 to 0 [417111.169418] md: md3 stopped. [417111.209370] md11: detected capacity change from 64011422924800 to 0 [417111.215740] md: md11 stopped. [417115.115260] LNet: Removed LNI 10.0.10.106@o2ib7 [417926.976141] md: md3 stopped. [417927.033793] md/raid:md3: device dm-56 operational as raid disk 0 [417927.039908] md/raid:md3: device dm-51 operational as raid disk 9 [417927.046014] md/raid:md3: device dm-50 operational as raid disk 8 [417927.052145] md/raid:md3: device dm-38 operational as raid disk 7 [417927.058256] md/raid:md3: device dm-37 operational as raid disk 6 [417927.064364] md/raid:md3: device dm-25 operational as raid disk 5 [417927.070465] md/raid:md3: device dm-24 operational as raid disk 4 [417927.076566] md/raid:md3: device dm-11 operational as raid disk 3 [417927.082668] md/raid:md3: device dm-10 operational as raid disk 2 [417927.088767] md/raid:md3: device dm-57 operational as raid disk 1 [417927.095789] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [417927.114918] md3: detected capacity change from 0 to 64011422924800 [417927.126161] md: md5 stopped. [417927.140917] md/raid:md5: device dm-2 operational as raid disk 0 [417927.146927] md/raid:md5: device dm-55 operational as raid disk 9 [417927.153030] md/raid:md5: device dm-54 operational as raid disk 8 [417927.159135] md/raid:md5: device dm-42 operational as raid disk 7 [417927.165233] md/raid:md5: device dm-41 operational as raid disk 6 [417927.171333] md/raid:md5: device dm-29 operational as raid disk 5 [417927.177462] md/raid:md5: device dm-28 operational as raid disk 4 [417927.183563] md/raid:md5: device dm-16 operational as raid disk 3 [417927.189663] md/raid:md5: device dm-15 operational as raid disk 2 [417927.195758] md/raid:md5: device dm-3 operational as raid disk 1 [417927.202599] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [417927.219057] md5: detected capacity change from 0 to 64011422924800 [417927.274442] md: md11 stopped. [417927.299283] md/raid:md11: device dm-62 operational as raid disk 0 [417927.305482] md/raid:md11: device dm-115 operational as raid disk 9 [417927.311760] md/raid:md11: device dm-114 operational as raid disk 8 [417927.318033] md/raid:md11: device dm-102 operational as raid disk 7 [417927.324308] md/raid:md11: device dm-101 operational as raid disk 6 [417927.330584] md/raid:md11: device dm-89 operational as raid disk 5 [417927.336771] md/raid:md11: device dm-88 operational as raid disk 4 [417927.342955] md/raid:md11: device dm-76 operational as raid disk 3 [417927.349143] md/raid:md11: device dm-75 operational as raid disk 2 [417927.355329] md/raid:md11: device dm-63 operational as raid disk 1 [417927.362341] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [417927.381242] md11: detected capacity change from 0 to 64011422924800 [417927.399468] md: md7 stopped. [417927.460485] md/raid:md7: device dm-72 operational as raid disk 0 [417927.466590] md/raid:md7: device dm-107 operational as raid disk 9 [417927.472781] md/raid:md7: device dm-106 operational as raid disk 8 [417927.478971] md/raid:md7: device dm-93 operational as raid disk 7 [417927.485082] md/raid:md7: device dm-92 operational as raid disk 6 [417927.491180] md/raid:md7: device dm-80 operational as raid disk 5 [417927.497281] md/raid:md7: device dm-79 operational as raid disk 4 [417927.503381] md/raid:md7: device dm-67 operational as raid disk 3 [417927.509485] md/raid:md7: device dm-66 operational as raid disk 2 [417927.515596] md/raid:md7: device dm-83 operational as raid disk 1 [417927.532156] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [417927.564542] md7: detected capacity change from 0 to 64011422924800 [417927.577687] md: md9 stopped. [417927.609368] md/raid:md9: device dm-116 operational as raid disk 0 [417927.615574] md/raid:md9: device dm-111 operational as raid disk 9 [417927.621771] md/raid:md9: device dm-110 operational as raid disk 8 [417927.627965] md/raid:md9: device dm-98 operational as raid disk 7 [417927.634081] md/raid:md9: device dm-97 operational as raid disk 6 [417927.640204] md/raid:md9: device dm-85 operational as raid disk 5 [417927.646310] md/raid:md9: device dm-84 operational as raid disk 4 [417927.652410] md/raid:md9: device dm-71 operational as raid disk 3 [417927.658521] md/raid:md9: device dm-70 operational as raid disk 2 [417927.664634] md/raid:md9: device dm-117 operational as raid disk 1 [417927.671584] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [417927.705970] md9: detected capacity change from 0 to 64011422924800 [417927.722044] md: md1 stopped. [417927.767723] md/raid:md1: device dm-12 operational as raid disk 0 [417927.773822] md/raid:md1: device dm-47 operational as raid disk 9 [417927.779927] md/raid:md1: device dm-46 operational as raid disk 8 [417927.786032] md/raid:md1: device dm-33 operational as raid disk 7 [417927.792139] md/raid:md1: device dm-32 operational as raid disk 6 [417927.798244] md/raid:md1: device dm-20 operational as raid disk 5 [417927.804354] md/raid:md1: device dm-19 operational as raid disk 4 [417927.810452] md/raid:md1: device dm-7 operational as raid disk 3 [417927.816466] md/raid:md1: device dm-6 operational as raid disk 2 [417927.822484] md/raid:md1: device dm-23 operational as raid disk 1 [417927.854982] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [417927.902507] md1: detected capacity change from 0 to 64011422924800 [417946.075593] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [417946.396206] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417946.936629] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [417947.222647] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [417947.262445] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417947.550875] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417947.553660] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [417947.932959] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417949.615682] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [417949.939278] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417953.805778] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [417954.142554] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417978.553788] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [417978.561477] alg: No test for adler32 (adler32-zlib) [417979.377489] Lustre: Lustre: Build Version: 2.12.0 [417979.511777] LNet: Using FastReg for registration [417979.527523] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [417984.449577] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [417984.733573] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [417984.815005] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417985.076348] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417985.099515] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [417985.219496] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [417985.433993] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417985.527887] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417985.554582] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [417985.585667] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [417985.659506] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [417985.894811] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417985.917397] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417985.959590] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [417985.969391] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417985.989499] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [417986.281565] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [417986.296369] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417986.424525] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [417986.528520] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [417986.727771] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417986.827863] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417986.972546] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [417986.981044] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [417987.008428] Lustre: fir-OST0019: new disk, initializing [417987.273414] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [417988.013523] Lustre: srv-fir-OST0019: No data found on store. Initialize space [417988.050973] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [417988.356211] Lustre: fir-OST0021: new disk, initializing [417988.900236] Lustre: srv-fir-OST001f: No data found on store. Initialize space [417988.907461] Lustre: Skipped 2 previous similar messages [417988.923749] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [417988.932061] Lustre: Skipped 2 previous similar messages [417992.322169] Lustre: fir-OST0019: Connection restored to (at 10.0.10.51@o2ib7) [417992.329483] Lustre: Skipped 1 previous similar message [417992.339624] Lustre: cli-fir-OST0021-super: Allocated super-sequence [0x00000004c0000400-0x0000000500000400]:21:ost] [417994.013211] Lustre: fir-OST001b: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [417994.023486] Lustre: Skipped 9 previous similar messages [418018.858075] Lustre: fir-OST0021: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [418018.858076] Lustre: fir-OST001f: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [418018.858080] Lustre: Skipped 13 previous similar messages [418018.883988] Lustre: Skipped 1 previous similar message [418056.218102] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [418056.235327] LustreError: Skipped 6 previous similar messages [418057.633530] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [418057.650994] LustreError: Skipped 5 previous similar messages [418081.218910] Lustre: fir-OST0019: Connection restored to (at 10.9.0.2@o2ib4) [418081.226061] Lustre: Skipped 7 previous similar messages [418089.564929] Lustre: fir-MDT0001-lwp-OST0019: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [418089.564930] Lustre: fir-MDT0003-lwp-OST0019: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [418089.564933] Lustre: Skipped 1 previous similar message [418089.602320] Lustre: Skipped 18 previous similar messages [418093.852164] Lustre: Failing over fir-OST0023 [418093.856529] Lustre: Skipped 3 previous similar messages [418094.017066] Lustre: fir-OST0023: Not available for connect from 10.8.9.6@o2ib6 (stopping) [418094.017067] Lustre: fir-OST001b: Not available for connect from 10.8.9.6@o2ib6 (stopping) [418094.033593] Lustre: Skipped 1 previous similar message [418094.607836] Lustre: server umount fir-OST001b complete [418095.144792] Lustre: server umount fir-OST001d complete [418095.150021] Lustre: Skipped 3 previous similar messages [418095.996962] md5: detected capacity change from 64011422924800 to 0 [418096.003243] md: md5 stopped. [418096.007335] md9: detected capacity change from 64011422924800 to 0 [418096.013616] md: md9 stopped. [418096.016739] md1: detected capacity change from 64011422924800 to 0 [418096.018288] md7: detected capacity change from 64011422924800 to 0 [418096.018296] md: md7 stopped. [418096.024624] md11: detected capacity change from 64011422924800 to 0 [418096.024627] md: md11 stopped. [418096.024917] md3: detected capacity change from 64011422924800 to 0 [418096.024924] md: md3 stopped. [418096.050902] md: md1 stopped. [418099.542984] LNet: Removed LNI 10.0.10.106@o2ib7 [418130.194603] md: md5 stopped. [418130.245433] md/raid:md5: device dm-2 operational as raid disk 0 [418130.251445] md/raid:md5: device dm-55 operational as raid disk 9 [418130.257539] md/raid:md5: device dm-54 operational as raid disk 8 [418130.263633] md/raid:md5: device dm-42 operational as raid disk 7 [418130.269733] md/raid:md5: device dm-41 operational as raid disk 6 [418130.275834] md/raid:md5: device dm-29 operational as raid disk 5 [418130.281928] md/raid:md5: device dm-28 operational as raid disk 4 [418130.288030] md/raid:md5: device dm-16 operational as raid disk 3 [418130.294131] md/raid:md5: device dm-15 operational as raid disk 2 [418130.300232] md/raid:md5: device dm-3 operational as raid disk 1 [418130.307096] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [418130.324773] md5: detected capacity change from 0 to 64011422924800 [418130.332137] md: md7 stopped. [418130.362794] md/raid:md7: device dm-72 operational as raid disk 0 [418130.368911] md/raid:md7: device dm-107 operational as raid disk 9 [418130.375103] md/raid:md7: device dm-106 operational as raid disk 8 [418130.381296] md/raid:md7: device dm-93 operational as raid disk 7 [418130.387406] md/raid:md7: device dm-92 operational as raid disk 6 [418130.393512] md/raid:md7: device dm-80 operational as raid disk 5 [418130.399614] md/raid:md7: device dm-79 operational as raid disk 4 [418130.405715] md/raid:md7: device dm-67 operational as raid disk 3 [418130.411813] md/raid:md7: device dm-66 operational as raid disk 2 [418130.417919] md/raid:md7: device dm-83 operational as raid disk 1 [418130.424936] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [418130.443430] md7: detected capacity change from 0 to 64011422924800 [418130.451652] md: md9 stopped. [418130.486050] md/raid:md9: device dm-116 operational as raid disk 0 [418130.492245] md/raid:md9: device dm-111 operational as raid disk 9 [418130.498431] md/raid:md9: device dm-110 operational as raid disk 8 [418130.504615] md/raid:md9: device dm-98 operational as raid disk 7 [418130.510718] md/raid:md9: device dm-97 operational as raid disk 6 [418130.516823] md/raid:md9: device dm-85 operational as raid disk 5 [418130.522920] md/raid:md9: device dm-84 operational as raid disk 4 [418130.529024] md/raid:md9: device dm-71 operational as raid disk 3 [418130.535128] md/raid:md9: device dm-70 operational as raid disk 2 [418130.541232] md/raid:md9: device dm-117 operational as raid disk 1 [418130.548520] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [418130.567618] md9: detected capacity change from 0 to 64011422924800 [418130.576804] md: md1 stopped. [418130.633094] md/raid:md1: device dm-12 operational as raid disk 0 [418130.639201] md/raid:md1: device dm-47 operational as raid disk 9 [418130.645302] md/raid:md1: device dm-46 operational as raid disk 8 [418130.651405] md/raid:md1: device dm-33 operational as raid disk 7 [418130.657517] md/raid:md1: device dm-32 operational as raid disk 6 [418130.663628] md/raid:md1: device dm-20 operational as raid disk 5 [418130.669769] md/raid:md1: device dm-19 operational as raid disk 4 [418130.675924] md/raid:md1: device dm-7 operational as raid disk 3 [418130.681932] md/raid:md1: device dm-6 operational as raid disk 2 [418130.687967] md/raid:md1: device dm-23 operational as raid disk 1 [418130.695211] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [418130.728622] md1: detected capacity change from 0 to 64011422924800 [418130.740776] md: md3 stopped. [418130.787296] md/raid:md3: device dm-56 operational as raid disk 0 [418130.793423] md/raid:md3: device dm-51 operational as raid disk 9 [418130.799542] md/raid:md3: device dm-50 operational as raid disk 8 [418130.805653] md/raid:md3: device dm-38 operational as raid disk 7 [418130.811759] md/raid:md3: device dm-37 operational as raid disk 6 [418130.817861] md/raid:md3: device dm-25 operational as raid disk 5 [418130.823963] md/raid:md3: device dm-24 operational as raid disk 4 [418130.830066] md/raid:md3: device dm-11 operational as raid disk 3 [418130.836173] md/raid:md3: device dm-10 operational as raid disk 2 [418130.842276] md/raid:md3: device dm-57 operational as raid disk 1 [418130.849243] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [418130.868264] md3: detected capacity change from 0 to 64011422924800 [418130.878470] md: md11 stopped. [418130.929668] md/raid:md11: device dm-62 operational as raid disk 0 [418130.935858] md/raid:md11: device dm-115 operational as raid disk 9 [418130.942134] md/raid:md11: device dm-114 operational as raid disk 8 [418130.948406] md/raid:md11: device dm-102 operational as raid disk 7 [418130.954684] md/raid:md11: device dm-101 operational as raid disk 6 [418130.960955] md/raid:md11: device dm-89 operational as raid disk 5 [418130.967144] md/raid:md11: device dm-88 operational as raid disk 4 [418130.973332] md/raid:md11: device dm-76 operational as raid disk 3 [418130.979522] md/raid:md11: device dm-75 operational as raid disk 2 [418130.985711] md/raid:md11: device dm-63 operational as raid disk 1 [418130.992615] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [418131.010727] md11: detected capacity change from 0 to 64011422924800 [418149.146284] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [418149.478384] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418149.942306] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [418150.278033] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418151.165335] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [418151.496421] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418151.876356] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [418152.197944] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418153.094374] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [418153.419010] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418154.528404] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [418154.857367] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418189.315309] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [418189.323083] alg: No test for adler32 (adler32-zlib) [418190.130938] Lustre: Lustre: Build Version: 2.12.0 [418190.260090] LNet: Using FastReg for registration [418190.275933] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [418195.030358] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [418195.374055] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418195.476613] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [418195.778392] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [418195.816040] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418195.937358] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [418196.113183] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418196.219493] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [418196.238622] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418196.355361] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [418196.382438] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [418196.553057] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418196.659839] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418196.724673] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418196.734458] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [418196.810384] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [418197.068624] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [418197.110767] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418197.322407] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [418197.353386] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [418197.458186] Lustre: fir-OST001d: new disk, initializing [418197.625363] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418197.657329] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418198.384425] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [418198.463436] Lustre: srv-fir-OST001d: No data found on store. Initialize space [418198.554009] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [418198.684269] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [418198.940274] Lustre: fir-OST0023: new disk, initializing [418199.226097] Lustre: srv-fir-OST0019: No data found on store. Initialize space [418199.233325] Lustre: Skipped 1 previous similar message [418199.251112] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [418199.259419] Lustre: Skipped 1 previous similar message [418200.263144] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [418200.280342] LustreError: Skipped 5 previous similar messages [418200.700601] Lustre: fir-OST001d: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [418200.768273] Lustre: cli-fir-OST001d-super: Allocated super-sequence [0x0000000540000400-0x0000000580000400]:1d:ost] [418201.718850] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [418201.736334] LustreError: Skipped 5 previous similar messages [418205.133039] Lustre: fir-OST001d: Connection restored to fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) [418205.142349] Lustre: Skipped 11 previous similar messages [418220.154546] Lustre: fir-OST0019: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [418220.164834] Lustre: Skipped 11 previous similar messages [418225.286809] Lustre: fir-OST001b: Connection restored to (at 10.9.0.2@o2ib4) [418225.293957] Lustre: Skipped 5 previous similar messages [418245.087342] Lustre: fir-OST0021: Connection restored to 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) [418245.097605] Lustre: Skipped 4 previous similar messages [418270.155773] Lustre: fir-OST0023: Connection restored to 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) [418270.166034] Lustre: Skipped 10 previous similar messages [418515.177531] Lustre: fir-OST001b: Connection restored to 3d981e2a-06a8-218c-1aa5-680f19d3d6b1 (at 10.0.10.3@o2ib7) [418515.187876] Lustre: Skipped 5 previous similar messages [418575.732242] Lustre: fir-OST0019: Connection restored to bd488cd4-324a-aaa3-2569-ed85c1419cd4 (at 10.9.101.59@o2ib4) [418575.742771] Lustre: Skipped 5 previous similar messages [419071.705439] Lustre: 123460:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548196757/real 1548196757] req@ffff9273d5048f00 x1623401045183760/t0(0) o39->fir-MDT0002-lwp-OST0021@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196763 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [419071.705441] Lustre: 123461:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548196757/real 1548196757] req@ffff926fbdd39500 x1623401045183776/t0(0) o39->fir-MDT0002-lwp-OST001b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196763 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [419071.762363] Lustre: 123460:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [419077.706575] Lustre: 123461:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548196763/real 1548196763] req@ffff926fbdd39500 x1623401045183856/t0(0) o39->fir-MDT0000-lwp-OST001b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196769 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [419077.735055] Lustre: 123461:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [419079.279614] LNet: 112390:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 4 seconds [419079.289918] Lustre: 112468:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548196769/real 1548196771] req@ffff9293eb911e00 x1623401045184096/t0(0) o400->fir-MDT0001-lwp-OST0019@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196776 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [419079.289937] Lustre: fir-MDT0001-lwp-OST001d: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [419079.334902] Lustre: 112468:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [419080.279640] LNet: 112390:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 223 seconds [419080.290070] LNet: 112390:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [419080.301451] Lustre: Failing over fir-OST001b [419080.305820] Lustre: Skipped 1 previous similar message [419081.123751] Lustre: server umount fir-OST001b complete [419081.281576] Lustre: Failing over fir-OST0023 [419081.285942] Lustre: Skipped 2 previous similar messages [419081.533243] md9: detected capacity change from 64011422924800 to 0 [419081.539528] md: md9 stopped. [419081.714802] md3: detected capacity change from 64011422924800 to 0 [419081.721082] md: md3 stopped. [419082.059800] Lustre: server umount fir-OST0023 complete [419082.065037] Lustre: Skipped 2 previous similar messages [419082.343433] md7: detected capacity change from 64011422924800 to 0 [419082.349717] md: md7 stopped. [419082.377896] md1: detected capacity change from 64011422924800 to 0 [419082.384180] md: md1 stopped. [419082.387909] md11: detected capacity change from 64011422924800 to 0 [419082.394279] md: md11 stopped. [419083.279726] LNet: 112390:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [419083.289980] LNet: 112390:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 9 previous similar messages [419083.299515] Lustre: 123459:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548196769/real 1548196775] req@ffff9293eb913600 x1623401045183952/t0(0) o39->fir-MDT0003-lwp-OST001d@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548196775 ref 2 fl Rpc:eXN/0/ffffffff rc 0/-1 [419083.328355] Lustre: 123459:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [419083.341970] Lustre: Failing over fir-OST001d [419084.159048] Lustre: server umount fir-OST001d complete [419084.164284] Lustre: Skipped 1 previous similar message [419084.410324] md5: detected capacity change from 64011422924800 to 0 [419084.416604] md: md5 stopped. [419088.309813] LNet: Removed LNI 10.0.10.106@o2ib7 [419106.459838] md: md9 stopped. [419106.511106] md/raid:md9: device dm-116 operational as raid disk 0 [419106.517293] md/raid:md9: device dm-111 operational as raid disk 9 [419106.523482] md/raid:md9: device dm-110 operational as raid disk 8 [419106.529669] md/raid:md9: device dm-98 operational as raid disk 7 [419106.535770] md/raid:md9: device dm-97 operational as raid disk 6 [419106.541864] md/raid:md9: device dm-85 operational as raid disk 5 [419106.547964] md/raid:md9: device dm-84 operational as raid disk 4 [419106.554058] md/raid:md9: device dm-71 operational as raid disk 3 [419106.560159] md/raid:md9: device dm-70 operational as raid disk 2 [419106.566261] md/raid:md9: device dm-117 operational as raid disk 1 [419106.573313] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [419106.590357] md9: detected capacity change from 0 to 64011422924800 [419106.602895] md: md1 stopped. [419106.659104] md/raid:md1: device dm-12 operational as raid disk 0 [419106.665241] md/raid:md1: device dm-47 operational as raid disk 9 [419106.671350] md/raid:md1: device dm-46 operational as raid disk 8 [419106.677460] md/raid:md1: device dm-33 operational as raid disk 7 [419106.683560] md/raid:md1: device dm-32 operational as raid disk 6 [419106.689686] md/raid:md1: device dm-20 operational as raid disk 5 [419106.695788] md/raid:md1: device dm-19 operational as raid disk 4 [419106.701892] md/raid:md1: device dm-7 operational as raid disk 3 [419106.707907] md/raid:md1: device dm-6 operational as raid disk 2 [419106.713919] md/raid:md1: device dm-23 operational as raid disk 1 [419106.720859] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [419106.739876] md1: detected capacity change from 0 to 64011422924800 [419106.748573] md: md7 stopped. [419106.772303] md/raid:md7: device dm-72 operational as raid disk 0 [419106.778401] md/raid:md7: device dm-107 operational as raid disk 9 [419106.784593] md/raid:md7: device dm-106 operational as raid disk 8 [419106.790800] md/raid:md7: device dm-93 operational as raid disk 7 [419106.796902] md/raid:md7: device dm-92 operational as raid disk 6 [419106.803006] md/raid:md7: device dm-80 operational as raid disk 5 [419106.809113] md/raid:md7: device dm-79 operational as raid disk 4 [419106.815214] md/raid:md7: device dm-67 operational as raid disk 3 [419106.821357] md/raid:md7: device dm-66 operational as raid disk 2 [419106.827454] md/raid:md7: device dm-83 operational as raid disk 1 [419106.834347] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [419106.853029] md7: detected capacity change from 0 to 64011422924800 [419106.871059] md: md5 stopped. [419106.923225] md/raid:md5: device dm-2 operational as raid disk 0 [419106.929244] md/raid:md5: device dm-55 operational as raid disk 9 [419106.935346] md/raid:md5: device dm-54 operational as raid disk 8 [419106.941446] md/raid:md5: device dm-42 operational as raid disk 7 [419106.947553] md/raid:md5: device dm-41 operational as raid disk 6 [419106.953657] md/raid:md5: device dm-29 operational as raid disk 5 [419106.959757] md/raid:md5: device dm-28 operational as raid disk 4 [419106.965858] md/raid:md5: device dm-16 operational as raid disk 3 [419106.971963] md/raid:md5: device dm-15 operational as raid disk 2 [419106.978064] md/raid:md5: device dm-3 operational as raid disk 1 [419106.985171] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [419107.009497] md5: detected capacity change from 0 to 64011422924800 [419107.036370] md: md11 stopped. [419107.104316] md/raid:md11: device dm-62 operational as raid disk 0 [419107.110498] md/raid:md11: device dm-115 operational as raid disk 9 [419107.116769] md/raid:md11: device dm-114 operational as raid disk 8 [419107.123061] md/raid:md11: device dm-102 operational as raid disk 7 [419107.129338] md/raid:md11: device dm-101 operational as raid disk 6 [419107.135653] md/raid:md11: device dm-89 operational as raid disk 5 [419107.141844] md/raid:md11: device dm-88 operational as raid disk 4 [419107.148074] md/raid:md11: device dm-76 operational as raid disk 3 [419107.154268] md/raid:md11: device dm-75 operational as raid disk 2 [419107.160494] md/raid:md11: device dm-63 operational as raid disk 1 [419107.167912] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [419107.187292] md11: detected capacity change from 0 to 64011422924800 [419107.195154] md: md3 stopped. [419107.252169] md/raid:md3: device dm-56 operational as raid disk 0 [419107.258303] md/raid:md3: device dm-51 operational as raid disk 9 [419107.264415] md/raid:md3: device dm-50 operational as raid disk 8 [419107.270521] md/raid:md3: device dm-38 operational as raid disk 7 [419107.276626] md/raid:md3: device dm-37 operational as raid disk 6 [419107.282738] md/raid:md3: device dm-25 operational as raid disk 5 [419107.288850] md/raid:md3: device dm-24 operational as raid disk 4 [419107.294960] md/raid:md3: device dm-11 operational as raid disk 3 [419107.301061] md/raid:md3: device dm-10 operational as raid disk 2 [419107.307164] md/raid:md3: device dm-57 operational as raid disk 1 [419107.314303] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [419107.333767] md3: detected capacity change from 0 to 64011422924800 [419125.781832] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [419126.107397] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419126.578846] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [419126.913847] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419126.991897] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [419127.317325] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419127.392868] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [419127.455879] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [419127.719463] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419127.779042] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419129.325921] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [419129.653370] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419216.992377] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [419217.000183] alg: No test for adler32 (adler32-zlib) [419217.805328] Lustre: Lustre: Build Version: 2.12.0 [419217.934396] LNet: Using FastReg for registration [419217.950092] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [419222.672177] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [419222.945158] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [419223.020765] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419223.225146] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [419223.291783] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419223.496870] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [419223.526094] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [419223.547734] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419223.837192] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419223.840122] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419223.936175] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [419223.995097] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [419224.209098] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [419224.250554] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [419224.281572] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419224.300965] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419224.488093] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [419224.510015] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419224.582074] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [419224.790124] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419225.226197] Lustre: fir-OST001d: new disk, initializing [419225.337123] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [419225.371119] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [419225.643822] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419225.671943] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [419226.231119] Lustre: srv-fir-OST001d: No data found on store. Initialize space [419226.273749] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [419226.557274] Lustre: fir-OST001f: new disk, initializing [419226.870657] Lustre: srv-fir-OST0019: No data found on store. Initialize space [419226.877886] Lustre: Skipped 2 previous similar messages [419226.893181] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [419226.901453] Lustre: Skipped 2 previous similar messages [419228.097945] Lustre: fir-OST001d: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [419228.107256] Lustre: Skipped 1 previous similar message [419228.127255] Lustre: cli-fir-OST001f-super: Allocated super-sequence [0x00000005c0000400-0x0000000600000400]:1f:ost] [419229.033367] Lustre: fir-OST001d: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [419229.042714] Lustre: Skipped 9 previous similar messages [419234.223315] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [419234.240519] LustreError: Skipped 5 previous similar messages [419239.401198] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [419239.418395] LustreError: Skipped 11 previous similar messages [419240.843627] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [419240.861096] LustreError: Skipped 3 previous similar messages [419242.296052] Lustre: fir-OST0019: Connection restored to 31582059-27cb-3e3b-d317-11d93512a873 (at 10.0.10.3@o2ib7) [419242.296053] Lustre: fir-OST001b: Connection restored to 31582059-27cb-3e3b-d317-11d93512a873 (at 10.0.10.3@o2ib7) [419242.296056] Lustre: Skipped 10 previous similar messages [419242.322197] Lustre: Skipped 4 previous similar messages [419259.224006] Lustre: fir-OST0019: Connection restored to (at 10.8.9.6@o2ib6) [419259.224007] Lustre: fir-OST001b: Connection restored to (at 10.8.9.6@o2ib6) [419259.238304] Lustre: Skipped 1 previous similar message [419295.458226] Lustre: fir-OST001f: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [419295.458227] Lustre: fir-OST0023: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [419295.458229] Lustre: fir-OST001d: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [419295.458231] Lustre: Skipped 23 previous similar messages [419295.458235] Lustre: Skipped 23 previous similar messages [419295.500622] Lustre: Skipped 1 previous similar message [420573.706157] Lustre: fir-OST001f: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [420573.716695] Lustre: Skipped 1 previous similar message [422806.336738] Lustre: fir-OST001b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [422806.347260] Lustre: Skipped 4 previous similar messages [423108.719151] Lustre: fir-OST0021: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [423108.729495] Lustre: Skipped 4 previous similar messages [423353.014399] Lustre: fir-OST001b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [423363.487462] Lustre: fir-OST001b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [424728.693660] Lustre: fir-OST0023: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [426329.828027] Lustre: fir-OST0019: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [426329.838372] Lustre: Skipped 4 previous similar messages [426331.091288] Lustre: fir-OST0021: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [426331.101640] Lustre: Skipped 1 previous similar message [426638.668442] Lustre: fir-OST001f: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [426638.678803] Lustre: Skipped 1 previous similar message [428450.073059] Lustre: fir-OST0019: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [428450.083410] Lustre: Skipped 5 previous similar messages [428606.333400] Lustre: fir-OST001b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [428606.343780] Lustre: Skipped 5 previous similar messages [430085.428554] Lustre: fir-OST001b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [430708.535038] Lustre: fir-OST0021: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [434833.481439] Lustre: fir-OST0019: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [434833.491807] Lustre: Skipped 2 previous similar messages [480459.091714] Lustre: fir-OST001b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [480600.237494] Lustre: fir-OST001d: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [480600.247844] Lustre: Skipped 3 previous similar messages [491457.727941] Lustre: fir-OST0019: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [491457.738483] Lustre: Skipped 8 previous similar messages [491878.327805] Lustre: fir-OST0019: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [491878.338326] Lustre: Skipped 5 previous similar messages [493480.452978] Lustre: fir-OST001b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [493535.479365] Lustre: fir-OST0019: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [493790.708345] Lustre: fir-OST0023: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494056.889145] Lustre: fir-OST0021: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494088.762649] Lustre: fir-OST001d: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494121.174289] Lustre: fir-OST001f: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494214.116769] Lustre: fir-OST001b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [494665.088855] Lustre: fir-OST0023: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [494970.344003] Lustre: fir-OST001b: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [494970.354351] Lustre: Skipped 2 previous similar messages [495138.053794] Lustre: fir-OST0019: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [495138.064314] Lustre: Skipped 1 previous similar message [500499.798037] Lustre: fir-OST0019: haven't heard from client 720fd129-ff4a-5f86-2ea9-26cf7b2bc80c (at 10.8.9.6@o2ib6) in 230 seconds. I think it's dead, and I am evicting it. exp ffff928458ff1800, cur 1548278189 expire 1548278039 last 1548277959 [504789.746267] Lustre: fir-OST001b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [504789.756795] Lustre: Skipped 11 previous similar messages [535337.001147] Lustre: fir-OST001b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [542254.073620] Lustre: fir-OST0021: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [542271.631804] Lustre: fir-OST001d: Connection restored to d27c605c-7097-4495-91c4-38ab61b2337f (at 10.8.6.33@o2ib6) [570980.815484] Lustre: fir-OST001d: haven't heard from client 35a268ae-728f-c804-8aa7-43bff9e699f9 (at 10.8.6.4@o2ib6) in 230 seconds. I think it's dead, and I am evicting it. exp ffff9273b54cc800, cur 1548348669 expire 1548348519 last 1548348439 [570980.837114] Lustre: Skipped 5 previous similar messages [578825.675087] Lustre: fir-OST001b: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [578825.685613] Lustre: Skipped 1 previous similar message [578888.029545] Lustre: fir-OST0019: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [578888.040072] Lustre: Skipped 1 previous similar message [579008.269738] Lustre: fir-OST0021: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [580171.128789] Lustre: fir-OST001d: Connection restored to 8f8627f4-8bd1-d857-1ae1-812035f146bb (at 10.9.101.59@o2ib4) [580171.139319] Lustre: Skipped 1 previous similar message [588542.042571] Lustre: fir-OST0019: Connection restored to 87096505-c370-0b84-8d41-ce56748694c4 (at 10.9.0.1@o2ib4) [588542.052850] Lustre: Skipped 4 previous similar messages [589030.311256] Lustre: fir-OST001b: Connection restored to 31582059-27cb-3e3b-d317-11d93512a873 (at 10.0.10.3@o2ib7) [589030.321608] Lustre: Skipped 5 previous similar messages [589503.865738] LNet: 127822:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 9 seconds [589503.876055] Lustre: 127866:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548367191/real 1548367191] req@ffff926cde3d0600 x1623402124773920/t0(0) o400->fir-MDT0003-lwp-OST0019@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548367198 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [589503.876069] Lustre: fir-MDT0001-lwp-OST001b: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [589503.921077] Lustre: 127866:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [589504.865768] LNet: 127822:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 121 seconds [589504.876193] LNet: 127822:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [589504.885726] Lustre: 127872:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548367191/real 1548367192] req@ffff926cde3d3f00 x1623402124774016/t0(0) o400->fir-MDT0003-lwp-OST0021@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548367198 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [589504.885734] Lustre: fir-MDT0001-lwp-OST0021: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [589504.885735] Lustre: Skipped 7 previous similar messages [589504.936291] Lustre: 127872:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 9 previous similar messages [589506.865819] LNet: 127822:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 12 seconds [589506.876162] LNet: 127822:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [589510.028889] Lustre: 45764:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548367191/real 1548367191] req@ffff9264747c1800 x1623402124774160/t0(0) o39->fir-MDT0000-lwp-OST0023@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548367197 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [589510.035343] Lustre: Failing over fir-OST001d [589510.061672] Lustre: 45764:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [589510.838378] Lustre: server umount fir-OST001f complete [589510.843639] Lustre: Skipped 4 previous similar messages [589511.396357] md5: detected capacity change from 64011422924800 to 0 [589511.399932] md3: detected capacity change from 64011422924800 to 0 [589511.399941] md: md3 stopped. [589511.411884] md: md5 stopped. [589511.466464] md7: detected capacity change from 64011422924800 to 0 [589511.472751] md: md7 stopped. [589511.476743] md1: detected capacity change from 64011422924800 to 0 [589511.477122] md9: detected capacity change from 64011422924800 to 0 [589511.477125] md: md9 stopped. [589511.492287] md: md1 stopped. [589511.680892] md11: detected capacity change from 64011422924800 to 0 [589511.687262] md: md11 stopped. [589515.989020] LNet: Removed LNI 10.0.10.106@o2ib7 [589591.982591] md: md7 stopped. [589591.994682] md/raid:md7: device dm-72 operational as raid disk 0 [589592.000781] md/raid:md7: device dm-107 operational as raid disk 9 [589592.006968] md/raid:md7: device dm-106 operational as raid disk 8 [589592.013155] md/raid:md7: device dm-93 operational as raid disk 7 [589592.019247] md/raid:md7: device dm-92 operational as raid disk 6 [589592.025342] md/raid:md7: device dm-80 operational as raid disk 5 [589592.031442] md/raid:md7: device dm-79 operational as raid disk 4 [589592.037535] md/raid:md7: device dm-67 operational as raid disk 3 [589592.043626] md/raid:md7: device dm-66 operational as raid disk 2 [589592.049721] md/raid:md7: device dm-83 operational as raid disk 1 [589592.056640] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [589592.075423] md7: detected capacity change from 0 to 64011422924800 [589592.085334] md: md9 stopped. [589592.101624] md/raid:md9: device dm-116 operational as raid disk 0 [589592.107811] md/raid:md9: device dm-111 operational as raid disk 9 [589592.113995] md/raid:md9: device dm-110 operational as raid disk 8 [589592.120184] md/raid:md9: device dm-98 operational as raid disk 7 [589592.126282] md/raid:md9: device dm-97 operational as raid disk 6 [589592.132377] md/raid:md9: device dm-85 operational as raid disk 5 [589592.138469] md/raid:md9: device dm-84 operational as raid disk 4 [589592.144563] md/raid:md9: device dm-71 operational as raid disk 3 [589592.150655] md/raid:md9: device dm-70 operational as raid disk 2 [589592.156749] md/raid:md9: device dm-117 operational as raid disk 1 [589592.163715] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [589592.181812] md9: detected capacity change from 0 to 64011422924800 [589592.189372] md: md5 stopped. [589592.208827] md/raid:md5: device dm-2 operational as raid disk 0 [589592.214857] md/raid:md5: device dm-55 operational as raid disk 9 [589592.220970] md/raid:md5: device dm-54 operational as raid disk 8 [589592.227084] md/raid:md5: device dm-42 operational as raid disk 7 [589592.233194] md/raid:md5: device dm-41 operational as raid disk 6 [589592.239303] md/raid:md5: device dm-29 operational as raid disk 5 [589592.245408] md/raid:md5: device dm-28 operational as raid disk 4 [589592.251507] md/raid:md5: device dm-16 operational as raid disk 3 [589592.257607] md/raid:md5: device dm-15 operational as raid disk 2 [589592.263709] md/raid:md5: device dm-3 operational as raid disk 1 [589592.270802] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [589592.289273] md5: detected capacity change from 0 to 64011422924800 [589592.305941] md: md3 stopped. [589592.350217] md/raid:md3: device dm-56 operational as raid disk 0 [589592.356322] md/raid:md3: device dm-51 operational as raid disk 9 [589592.362440] md/raid:md3: device dm-50 operational as raid disk 8 [589592.368558] md/raid:md3: device dm-38 operational as raid disk 7 [589592.374668] md/raid:md3: device dm-37 operational as raid disk 6 [589592.380778] md/raid:md3: device dm-25 operational as raid disk 5 [589592.386874] md/raid:md3: device dm-24 operational as raid disk 4 [589592.392974] md/raid:md3: device dm-11 operational as raid disk 3 [589592.399066] md/raid:md3: device dm-10 operational as raid disk 2 [589592.405161] md/raid:md3: device dm-57 operational as raid disk 1 [589592.412003] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [589592.430593] md3: detected capacity change from 0 to 64011422924800 [589592.440437] md: md11 stopped. [589592.474297] md/raid:md11: device dm-62 operational as raid disk 0 [589592.480496] md/raid:md11: device dm-115 operational as raid disk 9 [589592.486779] md/raid:md11: device dm-114 operational as raid disk 8 [589592.493063] md/raid:md11: device dm-102 operational as raid disk 7 [589592.499365] md/raid:md11: device dm-101 operational as raid disk 6 [589592.505651] md/raid:md11: device dm-89 operational as raid disk 5 [589592.511849] md/raid:md11: device dm-88 operational as raid disk 4 [589592.518044] md/raid:md11: device dm-76 operational as raid disk 3 [589592.524233] md/raid:md11: device dm-75 operational as raid disk 2 [589592.530420] md/raid:md11: device dm-63 operational as raid disk 1 [589592.538265] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [589592.557453] md11: detected capacity change from 0 to 64011422924800 [589592.585434] md: md1 stopped. [589592.611971] md/raid:md1: device dm-12 operational as raid disk 0 [589592.618075] md/raid:md1: device dm-47 operational as raid disk 9 [589592.624181] md/raid:md1: device dm-46 operational as raid disk 8 [589592.630285] md/raid:md1: device dm-33 operational as raid disk 7 [589592.636405] md/raid:md1: device dm-32 operational as raid disk 6 [589592.642505] md/raid:md1: device dm-20 operational as raid disk 5 [589592.648608] md/raid:md1: device dm-19 operational as raid disk 4 [589592.654737] md/raid:md1: device dm-7 operational as raid disk 3 [589592.660748] md/raid:md1: device dm-6 operational as raid disk 2 [589592.666765] md/raid:md1: device dm-23 operational as raid disk 1 [589592.673587] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [589592.693972] md1: detected capacity change from 0 to 64011422924800 [589611.767388] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [589612.111405] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [589612.155154] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589612.445577] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589612.519396] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [589612.884222] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589612.886399] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [589613.212110] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589614.158435] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [589614.487972] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589615.312471] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [589615.639596] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589682.626858] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [589682.634527] alg: No test for adler32 (adler32-zlib) [589683.438535] Lustre: Lustre: Build Version: 2.12.0 [589683.563283] LNet: Using FastReg for registration [589683.579633] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [589688.522203] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [589688.856628] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589689.018950] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [589689.083238] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [589689.358846] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589689.380159] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [589689.417885] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589689.551242] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [589689.683943] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589689.847020] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [589689.892946] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589690.111184] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [589690.127167] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [589690.186037] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [589690.191334] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589690.391173] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [589690.417369] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589690.434764] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589690.529514] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro [589690.690872] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589690.812847] Lustre: fir-OST001f: new disk, initializing [589690.975194] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [589690.983189] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [589691.279555] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589691.288424] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [589691.818206] Lustre: srv-fir-OST001f: No data found on store. Initialize space [589691.841729] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [589692.054347] Lustre: fir-OST0019: new disk, initializing [589692.289885] Lustre: fir-OST001f: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [589692.317106] Lustre: cli-fir-OST001f-super: Allocated super-sequence [0x0000000480000400-0x00000004c0000400]:1f:ost] [589692.327636] Lustre: Skipped 1 previous similar message [589692.332317] Lustre: srv-fir-OST0021: No data found on store. Initialize space [589692.332319] Lustre: Skipped 1 previous similar message [589692.342120] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [589692.342122] Lustre: Skipped 1 previous similar message [589693.571823] Lustre: fir-OST001f: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [589693.581135] Lustre: Skipped 3 previous similar messages [589693.590981] Lustre: cli-fir-OST0021-super: Allocated super-sequence [0x0000000980000400-0x00000009c0000400]:21:ost] [589699.136037] Lustre: fir-OST0021: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [589699.145347] Lustre: Skipped 11 previous similar messages [589700.116347] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [589700.133547] LustreError: Skipped 2 previous similar messages [589725.116974] Lustre: fir-OST0019: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [589725.127241] Lustre: Skipped 10 previous similar messages [589733.647456] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [589733.664949] LustreError: Skipped 3 previous similar messages [589758.648102] Lustre: fir-OST0019: Connection restored to (at 10.9.101.60@o2ib4) [589758.655510] Lustre: Skipped 2 previous similar messages [589783.648001] Lustre: fir-OST001f: Connection restored to (at 10.9.101.60@o2ib4) [589783.655405] Lustre: Skipped 2 previous similar messages [589904.969960] Lustre: fir-OST0019: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [589904.980489] Lustre: Skipped 10 previous similar messages [590205.868764] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [592587.842893] Lustre: fir-OST0019: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [592587.853418] Lustre: Skipped 10 previous similar messages [592767.128782] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [592767.139050] Lustre: Skipped 5 previous similar messages [592868.209044] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [599590.853244] Lustre: fir-OST0021: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [599590.863591] Lustre: Skipped 5 previous similar messages [599787.246317] Lustre: fir-OST001f: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [599787.256662] Lustre: Skipped 3 previous similar messages [600950.396732] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [600950.407094] Lustre: Skipped 3 previous similar messages [601241.148413] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [601241.158698] Lustre: Skipped 5 previous similar messages [601243.308131] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [601243.318393] Lustre: Skipped 1 previous similar message [601250.352825] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [601260.975204] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [601260.985465] Lustre: Skipped 5 previous similar messages [601280.695325] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [601280.705591] Lustre: Skipped 1 previous similar message [601542.246315] Lustre: fir-OST001f: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [604215.295229] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [604215.305488] Lustre: Skipped 6 previous similar messages [604884.921783] Lustre: fir-OST001b: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [604896.049027] Lustre: fir-OST0023: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [605229.539530] Lustre: fir-OST001b: Connection restored to e79c74e7-346c-5c0a-19ed-0389512537fa (at 10.9.101.59@o2ib4) [605229.550056] Lustre: Skipped 1 previous similar message [615650.769799] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [615950.800704] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [615950.811077] Lustre: Skipped 5 previous similar messages [616250.808266] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [616250.818616] Lustre: Skipped 5 previous similar messages [616550.817359] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [616550.827711] Lustre: Skipped 5 previous similar messages [616850.824182] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [616850.834526] Lustre: Skipped 5 previous similar messages [617150.833987] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [617150.844337] Lustre: Skipped 5 previous similar messages [617450.846160] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [617450.856508] Lustre: Skipped 5 previous similar messages [617750.848327] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [617750.858700] Lustre: Skipped 5 previous similar messages [618050.839733] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [618050.850079] Lustre: Skipped 5 previous similar messages [618350.864045] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [618350.874395] Lustre: Skipped 5 previous similar messages [618651.858781] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [618651.869127] Lustre: Skipped 5 previous similar messages [619251.873226] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [619251.883572] Lustre: Skipped 11 previous similar messages [619851.904157] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [619851.914500] Lustre: Skipped 11 previous similar messages [620750.930600] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [620750.940958] Lustre: Skipped 17 previous similar messages [621351.944620] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [621351.954969] Lustre: Skipped 11 previous similar messages [622251.968065] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [622251.978452] Lustre: Skipped 17 previous similar messages [623150.992743] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [623151.003091] Lustre: Skipped 17 previous similar messages [623752.010395] Lustre: fir-OST0019: Connection restored to c680654c-e7b5-e476-0d23-f02ec53e73ba (at 10.0.10.3@o2ib7) [623752.020746] Lustre: Skipped 11 previous similar messages [624378.057017] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [624378.067278] Lustre: Skipped 17 previous similar messages [643564.007597] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [643564.017858] Lustre: Skipped 13 previous similar messages [643763.872809] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [643763.883073] Lustre: Skipped 2 previous similar messages [645289.501909] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [645535.970606] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [645593.036298] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [645669.576286] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [646023.358914] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [646496.362762] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [648123.360305] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [648123.370565] Lustre: Skipped 1 previous similar message [648452.475363] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [648452.485623] Lustre: Skipped 1 previous similar message [648727.361910] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [649679.111044] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [649679.121311] Lustre: Skipped 1 previous similar message [649854.430601] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656134.750432] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656147.807329] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656147.817597] Lustre: Skipped 1 previous similar message [656181.676108] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656181.686375] Lustre: Skipped 2 previous similar messages [656231.946176] Lustre: fir-OST0019: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [656231.956449] Lustre: Skipped 8 previous similar messages [656307.414907] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656307.425171] Lustre: Skipped 4 previous similar messages [656478.912329] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656478.922599] Lustre: Skipped 10 previous similar messages [656805.558984] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [656805.569248] Lustre: Skipped 6 previous similar messages [657431.037014] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [657431.047288] Lustre: Skipped 13 previous similar messages [658063.926268] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [658063.936537] Lustre: Skipped 10 previous similar messages [658690.374665] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [658690.384927] Lustre: Skipped 11 previous similar messages [659363.308583] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [659363.318850] Lustre: Skipped 10 previous similar messages [659988.806604] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [659988.816867] Lustre: Skipped 9 previous similar messages [663373.151089] Lustre: fir-OST001f: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [663373.161359] Lustre: Skipped 8 previous similar messages [664682.995235] LNetError: 50660:0:(peer.c:2480:lnet_peer_merge_data()) Error deleting NID 10.10.6.33@tcp from peer 10.10.6.33@tcp: -16 [664684.012185] Lustre: fir-OST0019: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [664684.022458] Lustre: Skipped 5 previous similar messages [665234.599539] Lustre: fir-MDT0000-lwp-OST0019: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [665234.615644] Lustre: Skipped 10 previous similar messages [665262.308033] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665262.318245] Lustre: 50693:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548442945/real 1548442948] req@ffff9279cffd7200 x1623580866030112/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1548442952 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [665262.346731] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [665263.308045] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [665288.308619] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 20 seconds [665292.308710] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665292.318882] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 10 previous similar messages [665318.309310] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [665338.309768] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 2833 seconds [665367.310440] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665367.320614] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 11 previous similar messages [665417.311597] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665417.321766] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [665488.313223] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 221 seconds [665488.323566] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 13 previous similar messages [665618.316239] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [665618.326412] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 36 previous similar messages [665658.535055] Lustre: fir-OST001f: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [665658.544375] Lustre: Skipped 1 previous similar message [665688.754017] Lustre: Evicted from MGS (at 10.0.10.51@o2ib7) after server handle changed from 0xffdc3423dd164082 to 0x824cf209dbc07d1b [665688.766120] LustreError: 167-0: fir-MDT0002-lwp-OST001d: This client was evicted by fir-MDT0002; in progress operations using this service will fail. [665688.766157] Lustre: MGC10.0.10.51@o2ib7: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [665688.766158] Lustre: Skipped 10 previous similar messages [665709.654617] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:2664 to 0xd00000402:2689 [665709.654618] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:2664 to 0x980000402:2689 [665709.654619] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:2667 to 0x480000400:2689 [665709.654620] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:2666 to 0xd40000402:2689 [665709.654664] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:2665 to 0xcc0000402:2689 [665709.654697] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:2663 to 0x640000400:2689 [665709.660915] Lustre: fir-OST001b: deleting orphan objects from 0x0:6 to 0x0:33 [665709.660938] Lustre: fir-OST001f: deleting orphan objects from 0x0:5 to 0x0:33 [665709.660952] Lustre: fir-OST0019: deleting orphan objects from 0x0:5 to 0x0:33 [665709.660957] Lustre: fir-OST0021: deleting orphan objects from 0x0:6 to 0x0:33 [665709.660960] Lustre: fir-OST001d: deleting orphan objects from 0x0:6 to 0x0:33 [665709.661050] Lustre: fir-OST0023: deleting orphan objects from 0x0:5 to 0x0:33 [665711.303935] Lustre: fir-OST001b: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [665711.314200] Lustre: Skipped 12 previous similar messages [665722.059012] Lustre: fir-OST0023: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [665722.069278] Lustre: Skipped 1 previous similar message [665747.683209] Lustre: fir-OST001f: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [665799.155842] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [665829.959223] Lustre: fir-OST001d: Connection restored to 499c50f1-e88f-d71b-3734-bc99bfed13f5 (at 10.8.9.6@o2ib6) [665829.969499] Lustre: Skipped 2 previous similar messages [666090.809940] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [666167.616712] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [666503.794218] Lustre: fir-OST001d: Connection restored to 0f8221e3-c4fb-0a33-6bcf-5c53c20008ed (at 10.10.6.33@tcp) [666547.564477] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.10.6.33@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. [666890.172355] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [666890.182647] Lustre: Skipped 5 previous similar messages [666902.367644] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666902.378199] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444588/real 1548444588] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444599 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [666902.866875] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666902.877395] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 55179 previous similar messages [666902.888271] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444588/real 1548444588] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444599 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666902.915801] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 55179 previous similar messages [666903.866907] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666903.877424] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 112300 previous similar messages [666903.888905] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444589/real 1548444589] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444600 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666903.916425] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 112362 previous similar messages [666905.866946] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666905.877462] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 233998 previous similar messages [666905.888958] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444591/real 1548444591] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444602 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666905.916478] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 233999 previous similar messages [666909.867042] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666909.877561] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 474371 previous similar messages [666909.889051] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444595/real 1548444595] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444606 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666909.916569] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 474370 previous similar messages [666917.867224] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666917.877739] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 947736 previous similar messages [666917.889230] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444603/real 1548444603] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444614 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666917.916747] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 947737 previous similar messages [666933.867592] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666933.878116] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 1903613 previous similar messages [666933.889603] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444619/real 1548444619] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444630 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666933.917126] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1903601 previous similar messages [666965.868335] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [666965.878858] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 3813101 previous similar messages [666965.890338] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444651/real 1548444651] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444662 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [666965.917861] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3813101 previous similar messages [667029.869814] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [667029.880337] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 7698359 previous similar messages [667029.891816] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444715/real 1548444715] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444726 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [667029.919319] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7698358 previous similar messages [667157.872767] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [667157.883285] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 15401879 previous similar messages [667157.894769] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548444843/real 1548444843] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548444854 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [667157.922298] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 15401869 previous similar messages [667413.878682] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.6.33@tcp from [667413.889205] LNetError: 51008:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 30776590 previous similar messages [667413.900680] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548445099/real 1548445099] req@ffff92841b280900 x1623580866062544/t0(0) o105->fir-OST0023@10.10.6.33@tcp:15/16 lens 360/224 e 0 to 1 dl 1548445110 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [667413.928209] Lustre: 51008:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 30776589 previous similar messages [667497.211613] Lustre: 67508:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff927609daa700 x1623586290959328/t0(0) o103->3577bb83-18fa-14e9-4b08-b3335e8f2568@10.9.0.1@o2ib4:588/0 lens 328/224 e 24 to 0 dl 1548445188 ref 2 fl Interpret:H/0/0 rc 0/0 [667503.451756] LNet: Service thread pid 51008 was inactive for 601.07s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [667503.468773] Pid: 51008, comm: ldlm_cn02_001 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [667503.478939] Call Trace: [667503.481480] [] 0xffffffffffffffff [667503.486602] LustreError: dumping log to /tmp/lustre-log.1548445189.51008 [667521.360177] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 6 seconds [667521.370344] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [667521.379864] Lustre: fir-MDT0001-lwp-OST001d: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [667521.395934] Lustre: Skipped 7 previous similar messages [667522.360244] Lustre: fir-MDT0003-lwp-OST0023: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [667522.376306] Lustre: Skipped 3 previous similar messages [667526.360308] Lustre: fir-MDT0001-lwp-OST001f: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [667565.543744] Lustre: fir-OST0023: Client 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) reconnecting [667565.553776] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [667565.564033] Lustre: Skipped 3 previous similar messages [667576.541627] Lustre: fir-MDT0000-lwp-OST0019: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [667576.557709] Lustre: Skipped 11 previous similar messages [667577.361473] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 46 seconds [667577.371725] LNet: 50629:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 23 previous similar messages [667612.648636] Lustre: Failing over fir-OST0019 [667612.649700] LNet: Service thread pid 51008 completed after 710.26s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [667612.669277] Lustre: Skipped 5 previous similar messages [667613.424709] Lustre: server umount fir-OST001b complete [667613.974742] Lustre: server umount fir-OST0019 complete [667613.979977] Lustre: Skipped 4 previous similar messages [667619.397767] md11: detected capacity change from 64011422924800 to 0 [667619.404132] md: md11 stopped. [667619.406462] md3: detected capacity change from 64011422924800 to 0 [667619.406471] md: md3 stopped. [667619.450729] md9: detected capacity change from 64011422924800 to 0 [667619.457012] md: md9 stopped. [667620.497646] md1: detected capacity change from 64011422924800 to 0 [667620.503959] md: md1 stopped. [667620.622267] md5: detected capacity change from 64011422924800 to 0 [667620.628575] md: md5 stopped. [667620.663071] md7: detected capacity change from 64011422924800 to 0 [667620.669358] md: md7 stopped. [667624.422549] LNet: Removed LNI 10.0.10.106@o2ib7 [667717.871767] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [667717.879503] alg: No test for adler32 (adler32-zlib) [667718.682826] Lustre: Lustre: Build Version: 2.12.0 [667718.807197] LNet: Using FastReg for registration [667718.823737] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [667723.164461] md: md5 stopped. [667723.175904] md/raid:md5: device dm-2 operational as raid disk 0 [667723.181919] md/raid:md5: device dm-55 operational as raid disk 9 [667723.188040] md/raid:md5: device dm-54 operational as raid disk 8 [667723.194140] md/raid:md5: device dm-42 operational as raid disk 7 [667723.200270] md/raid:md5: device dm-41 operational as raid disk 6 [667723.206367] md/raid:md5: device dm-29 operational as raid disk 5 [667723.212499] md/raid:md5: device dm-28 operational as raid disk 4 [667723.218596] md/raid:md5: device dm-16 operational as raid disk 3 [667723.224698] md/raid:md5: device dm-15 operational as raid disk 2 [667723.230800] md/raid:md5: device dm-3 operational as raid disk 1 [667723.237622] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [667723.263836] md5: detected capacity change from 0 to 64011422924800 [667723.276719] md: md11 stopped. [667723.308054] md/raid:md11: device dm-62 operational as raid disk 0 [667723.314243] md/raid:md11: device dm-115 operational as raid disk 9 [667723.320524] md/raid:md11: device dm-114 operational as raid disk 8 [667723.326794] md/raid:md11: device dm-102 operational as raid disk 7 [667723.333069] md/raid:md11: device dm-101 operational as raid disk 6 [667723.339342] md/raid:md11: device dm-89 operational as raid disk 5 [667723.345524] md/raid:md11: device dm-88 operational as raid disk 4 [667723.351728] md/raid:md11: device dm-76 operational as raid disk 3 [667723.357916] md/raid:md11: device dm-75 operational as raid disk 2 [667723.364102] md/raid:md11: device dm-63 operational as raid disk 1 [667723.370994] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [667723.397240] md11: detected capacity change from 0 to 64011422924800 [667723.404765] md: md3 stopped. [667723.425261] md/raid:md3: device dm-56 operational as raid disk 0 [667723.431361] md/raid:md3: device dm-51 operational as raid disk 9 [667723.437463] md/raid:md3: device dm-50 operational as raid disk 8 [667723.443575] md/raid:md3: device dm-38 operational as raid disk 7 [667723.449676] md/raid:md3: device dm-37 operational as raid disk 6 [667723.455779] md/raid:md3: device dm-25 operational as raid disk 5 [667723.461877] md/raid:md3: device dm-24 operational as raid disk 4 [667723.467982] md/raid:md3: device dm-11 operational as raid disk 3 [667723.474084] md/raid:md3: device dm-10 operational as raid disk 2 [667723.480182] md/raid:md3: device dm-57 operational as raid disk 1 [667723.487117] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [667723.505511] md3: detected capacity change from 0 to 64011422924800 [667723.522413] md: md9 stopped. [667723.549232] md/raid:md9: device dm-116 operational as raid disk 0 [667723.555453] md/raid:md9: device dm-111 operational as raid disk 9 [667723.561657] md/raid:md9: device dm-110 operational as raid disk 8 [667723.567854] md/raid:md9: device dm-98 operational as raid disk 7 [667723.573961] md/raid:md9: device dm-97 operational as raid disk 6 [667723.580061] md/raid:md9: device dm-85 operational as raid disk 5 [667723.586165] md/raid:md9: device dm-84 operational as raid disk 4 [667723.592263] md/raid:md9: device dm-71 operational as raid disk 3 [667723.598366] md/raid:md9: device dm-70 operational as raid disk 2 [667723.604467] md/raid:md9: device dm-117 operational as raid disk 1 [667723.611596] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [667723.630733] md9: detected capacity change from 0 to 64011422924800 [667723.646419] md: md7 stopped. [667723.670498] md/raid:md7: device dm-72 operational as raid disk 0 [667723.676623] md/raid:md7: device dm-107 operational as raid disk 9 [667723.682806] md/raid:md7: device dm-106 operational as raid disk 8 [667723.689002] md/raid:md7: device dm-93 operational as raid disk 7 [667723.695121] md/raid:md7: device dm-92 operational as raid disk 6 [667723.701222] md/raid:md7: device dm-80 operational as raid disk 5 [667723.707347] md/raid:md7: device dm-79 operational as raid disk 4 [667723.713445] md/raid:md7: device dm-67 operational as raid disk 3 [667723.719564] md/raid:md7: device dm-66 operational as raid disk 2 [667723.725662] md/raid:md7: device dm-83 operational as raid disk 1 [667723.732479] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [667723.751069] md7: detected capacity change from 0 to 64011422924800 [667723.759839] md: md1 stopped. [667723.839382] md/raid:md1: device dm-12 operational as raid disk 0 [667723.845493] md/raid:md1: device dm-47 operational as raid disk 9 [667723.851604] md/raid:md1: device dm-46 operational as raid disk 8 [667723.857726] md/raid:md1: device dm-33 operational as raid disk 7 [667723.863850] md/raid:md1: device dm-32 operational as raid disk 6 [667723.869974] md/raid:md1: device dm-20 operational as raid disk 5 [667723.876123] md/raid:md1: device dm-19 operational as raid disk 4 [667723.882242] md/raid:md1: device dm-7 operational as raid disk 3 [667723.888270] md/raid:md1: device dm-6 operational as raid disk 2 [667723.894301] md/raid:md1: device dm-23 operational as raid disk 1 [667723.901579] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [667723.920318] md1: detected capacity change from 0 to 64011422924800 [667724.043034] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [667724.328277] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [667724.396457] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667724.660212] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667724.788064] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [667725.006099] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [667725.148698] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667725.154088] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [667725.361767] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667725.481760] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667725.812072] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [667726.127984] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [667726.806941] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [667727.678278] LustreError: 137-5: fir-OST001f_UUID: not available for connect from 10.8.6.33@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [667728.312974] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.0.10.3@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [667728.330250] LustreError: Skipped 10 previous similar messages [667729.560438] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [667729.577631] LustreError: Skipped 2 previous similar messages [667731.604996] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.8.9.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [667731.622199] LustreError: Skipped 9 previous similar messages [667732.807060] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [667734.807105] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [667734.817271] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [667736.809282] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [667736.818247] Lustre: fir-OST001d: in recovery but waiting for the first client to connect [667736.897145] Lustre: fir-OST001d: Will be in recovery for at least 5:00, or until 6 clients reconnect [667736.906404] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [667737.807196] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 5 seconds [667737.817363] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [667741.809344] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [667741.818260] Lustre: fir-OST0023: in recovery but waiting for the first client to connect [667741.911034] Lustre: fir-OST0023: Will be in recovery for at least 5:00, or until 6 clients reconnect [667741.920289] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [667742.807303] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 10 seconds [667742.817563] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [667743.554530] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [667743.564802] Lustre: Skipped 2 previous similar messages [667746.809422] Lustre: fir-OST001b: Imperative Recovery not enabled, recovery window 300-900 [667746.818981] Lustre: fir-OST001b: in recovery but waiting for the first client to connect [667747.698825] Lustre: fir-OST001b: Will be in recovery for at least 5:00, or until 6 clients reconnect [667747.708092] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [667749.497583] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [667749.514785] LustreError: Skipped 1 previous similar message [667751.807502] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 19 seconds [667751.817754] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [667751.829292] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [667751.838237] Lustre: fir-OST001f: in recovery but waiting for the first client to connect [667754.099714] Lustre: fir-OST001f: Will be in recovery for at least 5:00, or until 5 clients reconnect [667754.108980] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [667754.116212] Lustre: Skipped 3 previous similar messages [667756.809835] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [667756.827179] Lustre: fir-OST0021: in recovery but waiting for the first client to connect [667762.312226] Lustre: fir-OST0019: Will be in recovery for at least 5:00, or until 6 clients reconnect [667762.321449] Lustre: Skipped 1 previous similar message [667762.326728] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [667762.333949] Lustre: Skipped 9 previous similar messages [667764.626048] Lustre: fir-OST0023: Recovery over after 0:23, of 6 clients 6 recovered and 0 were evicted. [667765.149598] Lustre: fir-OST001f: Recovery over after 0:11, of 5 clients 5 recovered and 0 were evicted. [667765.247933] Lustre: fir-OST0019: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 6 known clients (2 recovered, 1 in progress, and 0 evicted) already passed deadline 0:02 [667765.266952] Lustre: Skipped 1 previous similar message [667766.217270] Lustre: fir-OST0019: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 6 known clients (2 recovered, 1 in progress, and 0 evicted) already passed deadline 0:03 [667766.236309] Lustre: Skipped 15 previous similar messages [667790.487891] Lustre: fir-OST0019: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 6 known clients (4 recovered, 1 in progress, and 0 evicted) already passed deadline 0:28 [667790.506912] Lustre: Skipped 19 previous similar messages [667815.576665] Lustre: fir-OST0019: Denying connection for new client fir-MDT0000-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 6 known clients (4 recovered, 1 in progress, and 0 evicted) already passed deadline 0:53 [667815.595682] Lustre: Skipped 15 previous similar messages [667816.350273] Lustre: fir-OST0023: deleting orphan objects from 0x0:68 to 0x0:97 [667816.364669] Lustre: fir-OST001f: deleting orphan objects from 0x0:67 to 0x0:97 [667816.602242] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:2731 to 0x480000400:2753 [667816.624884] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:2732 to 0xd40000402:2753 [667816.832746] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [667816.843016] Lustre: Skipped 19 previous similar messages [667816.907067] Lustre: fir-OST0019: Recovery over after 0:54, of 6 clients 6 recovered and 0 were evicted. [667817.451337] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:3 to 0x480000402:33 [667817.460006] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:3 to 0xd40000401:33 [667840.665977] Lustre: fir-OST0021: deleting orphan objects from 0x0:67 to 0x0:97 [667840.667787] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:2732 to 0xcc0000402:2753 [667840.668698] Lustre: fir-OST0019: deleting orphan objects from 0x0:67 to 0x0:97 [667840.668707] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:2733 to 0x640000400:2753 [667840.668738] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:2732 to 0xd00000402:2753 [667840.671786] Lustre: fir-OST001b: deleting orphan objects from 0x0:68 to 0x0:97 [667840.691701] Lustre: fir-OST001d: deleting orphan objects from 0x0:67 to 0x0:97 [667840.703218] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:2732 to 0x980000402:2753 [667913.463845] Lustre: fir-OST001b: Connection restored to (at 10.8.9.6@o2ib6) [667913.470984] Lustre: Skipped 22 previous similar messages [668180.378851] Lustre: fir-OST0019: Connection restored to (at 10.8.9.6@o2ib6) [668180.386092] Lustre: Skipped 1 previous similar message [668512.488096] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [668512.498353] Lustre: Skipped 14 previous similar messages [668858.476997] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [668858.487264] Lustre: Skipped 5 previous similar messages [669570.227661] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [669570.237928] Lustre: Skipped 3 previous similar messages [670221.678790] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [670221.689082] Lustre: Skipped 20 previous similar messages [670899.320857] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [670899.328095] Lustre: Skipped 2 previous similar messages [673282.686093] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [673282.693323] Lustre: Skipped 12 previous similar messages [673564.244250] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [673564.254521] Lustre: Skipped 4 previous similar messages [673716.935282] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [673716.945539] Lustre: Skipped 4 previous similar messages [674054.436699] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [674054.446969] Lustre: Skipped 5 previous similar messages [674769.029936] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [674769.040197] Lustre: Skipped 9 previous similar messages [675370.312835] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [675370.323097] Lustre: Skipped 12 previous similar messages [676158.341181] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [676158.348409] Lustre: Skipped 28 previous similar messages [676786.789430] Lustre: fir-OST001b: Connection restored to (at 10.8.9.6@o2ib6) [676786.796576] Lustre: Skipped 21 previous similar messages [677387.066894] Lustre: fir-OST001b: Connection restored to (at 10.8.9.6@o2ib6) [677387.074046] Lustre: Skipped 14 previous similar messages [678001.138831] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [678001.146062] Lustre: Skipped 16 previous similar messages [678616.689174] Lustre: fir-OST001d: Connection restored to (at 10.8.9.6@o2ib6) [678616.696326] Lustre: Skipped 23 previous similar messages [679223.549340] Lustre: fir-OST001d: Connection restored to (at 10.8.9.6@o2ib6) [679223.556486] Lustre: Skipped 20 previous similar messages [679847.105484] Lustre: fir-OST001f: Connection restored to (at 10.8.9.6@o2ib6) [679847.112631] Lustre: Skipped 31 previous similar messages [680466.494623] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [680466.501853] Lustre: Skipped 38 previous similar messages [681075.271156] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [681075.278384] Lustre: Skipped 34 previous similar messages [681677.922491] Lustre: fir-OST001f: Connection restored to (at 10.8.9.6@o2ib6) [681677.929631] Lustre: Skipped 24 previous similar messages [682345.300394] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [682345.310657] Lustre: Skipped 38 previous similar messages [682959.478798] Lustre: fir-OST001f: Connection restored to (at 10.8.9.6@o2ib6) [682959.486030] Lustre: Skipped 35 previous similar messages [683584.459944] Lustre: fir-OST001b: Connection restored to (at 10.8.9.6@o2ib6) [683584.467089] Lustre: Skipped 14 previous similar messages [685307.494791] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [685307.505055] Lustre: Skipped 2 previous similar messages [689734.901569] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [689734.908799] Lustre: Skipped 1 previous similar message [689806.255411] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [689947.531633] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [689947.538871] Lustre: Skipped 1 previous similar message [690196.535666] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [690507.704929] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [690507.712181] Lustre: Skipped 1 previous similar message [690703.408261] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [691061.345857] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [691061.353090] Lustre: Skipped 3 previous similar messages [691679.605824] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [691679.613054] Lustre: Skipped 13 previous similar messages [692487.869693] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [692487.876923] Lustre: Skipped 13 previous similar messages [693116.150052] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [693116.160317] Lustre: Skipped 12 previous similar messages [693743.057620] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [693743.067880] Lustre: Skipped 23 previous similar messages [694348.124051] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [694348.134320] Lustre: Skipped 19 previous similar messages [694992.105554] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [694992.112790] Lustre: Skipped 15 previous similar messages [695595.666779] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [695595.677042] Lustre: Skipped 15 previous similar messages [696225.032749] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [696225.043020] Lustre: Skipped 15 previous similar messages [696918.508028] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [696918.518290] Lustre: Skipped 18 previous similar messages [697520.048711] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [697520.058975] Lustre: Skipped 16 previous similar messages [698233.896690] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [698233.903916] Lustre: Skipped 20 previous similar messages [698878.446152] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [698878.453384] Lustre: Skipped 14 previous similar messages [699644.342140] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [699644.352403] Lustre: Skipped 13 previous similar messages [700310.584658] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [700310.591893] Lustre: Skipped 10 previous similar messages [700919.598801] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [700919.609065] Lustre: Skipped 13 previous similar messages [701539.033437] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [701539.043711] Lustre: Skipped 6 previous similar messages [702163.896660] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [702163.906923] Lustre: Skipped 6 previous similar messages [702769.306505] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [702769.316773] Lustre: Skipped 12 previous similar messages [702810.415728] Lustre: fir-OST0021: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [702810.424937] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:14189 to 0x980000400:14209 [702829.903955] Lustre: fir-OST001f: Client fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [702829.904243] Lustre: fir-OST001b: deleting orphan objects from 0xd00000400:14183 to 0xd00000400:14209 [702829.904307] Lustre: fir-OST0019: deleting orphan objects from 0x640000402:26028 to 0x640000402:26049 [702829.904356] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:26012 to 0xcc0000401:26049 [702829.904466] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:26026 to 0x980000401:26049 [702829.949879] Lustre: Skipped 10 previous similar messages [702863.165318] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [702863.165496] Lustre: fir-OST0019: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [702863.165668] Lustre: fir-OST0019: deleting orphan objects from 0x640000401:14188 to 0x640000401:14209 [702863.166811] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:26096 to 0x480000402:26113 [702863.210176] LustreError: Skipped 13 previous similar messages [702888.253890] Lustre: fir-OST001f: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [702888.254634] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:14189 to 0x980000400:14241 [702888.254972] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:14176 to 0xcc0000400:14209 [702888.255101] Lustre: fir-OST0023: deleting orphan objects from 0xd40000400:14176 to 0xd40000400:14209 [702888.256387] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:26097 to 0xd40000401:26113 [702888.299833] Lustre: Skipped 5 previous similar messages [702888.305309] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:14162 to 0x480000401:14177 [703393.070864] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [703393.078095] Lustre: Skipped 53 previous similar messages [704149.165827] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [704149.173057] Lustre: Skipped 11 previous similar messages [704772.370901] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [704772.381201] Lustre: Skipped 9 previous similar messages [706097.146042] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [706097.153275] Lustre: Skipped 7 previous similar messages [706185.290412] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [706185.297645] Lustre: Skipped 1 previous similar message [706833.434467] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [706833.444752] Lustre: Skipped 4 previous similar messages [707230.409913] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [707230.417202] Lustre: Skipped 1 previous similar message [710585.483405] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [710585.493676] Lustre: Skipped 8 previous similar messages [710767.421253] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [711141.783696] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [711917.716781] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [711917.727044] Lustre: Skipped 3 previous similar messages [712565.032614] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [712565.042877] Lustre: Skipped 3 previous similar messages [713429.956270] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [714155.869800] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [714155.880064] Lustre: Skipped 4 previous similar messages [715054.945242] Lustre: fir-OST0019: Client f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) reconnecting [715054.945266] Lustre: fir-OST001d: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [715054.945268] Lustre: Skipped 1 previous similar message [715054.970891] Lustre: Skipped 1 previous similar message [715175.631080] Lustre: fir-OST0021: Client 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) reconnecting [715175.641196] Lustre: Skipped 3 previous similar messages [715193.893497] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.3.11@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [715193.910781] LustreError: Skipped 4 previous similar messages [715218.953674] Lustre: fir-OST0019: Client 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) reconnecting [715218.963773] Lustre: Skipped 4 previous similar messages [718494.501281] Lustre: 61957:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1548496175/real 1548496179] req@ffff927c9289ce00 x1623662697349696/t0(0) o400->fir-MDT0000-lwp-OST0019@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548496182 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [718494.501291] Lustre: fir-MDT0002-lwp-OST0019: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [718494.546199] Lustre: 61957:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [718495.208154] Lustre: fir-OST001f: Client f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) reconnecting [718495.218287] Lustre: fir-OST001f: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [718495.228667] Lustre: Skipped 12 previous similar messages [718495.429175] LustreError: 72076:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff926412802850 x1623671563137632/t0(0) o4->f9afcafe-b7fa-076a-0d87-fc0f78fd02d0@10.8.6.20@o2ib6:245/0 lens 488/448 e 0 to 0 dl 1548496185 ref 1 fl Interpret:/0/0 rc 0/0 [718495.454065] Lustre: fir-OST001f: Bulk IO write error with f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6), client will retry: rc = -110 [718495.990070] Lustre: fir-OST001f: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [718497.869228] Lustre: 61956:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1548496175/real 1548496175] req@ffff927c9289bf00 x1623662697349488/t0(0) o400->fir-MDT0002-lwp-OST001b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1548496182 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [718497.869233] Lustre: fir-MDT0000-lwp-OST0023: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [718497.869236] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [718497.869237] Lustre: Skipped 7 previous similar messages [718497.932954] Lustre: 61956:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [718499.979278] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [718499.989360] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (4): c: 0, oc: 2, rc: 8 [718500.193365] Lustre: fir-OST001f: Client f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) reconnecting [718500.454289] LustreError: 64328:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff927abb934c50 x1623671563137632/t0(0) o4->f9afcafe-b7fa-076a-0d87-fc0f78fd02d0@10.8.6.20@o2ib6:271/0 lens 488/448 e 1 to 0 dl 1548496211 ref 1 fl Interpret:/2/0 rc 0/0 [718500.479054] Lustre: fir-OST001f: Bulk IO write error with f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6), client will retry: rc = -110 [718507.677325] Lustre: fir-OST0023: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [718507.678061] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:43918 to 0x480000400:43937 [718507.678264] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:43904 to 0xd40000402:43937 [718507.678277] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:43900 to 0xd00000402:43937 [718507.678292] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:43923 to 0xcc0000402:43969 [718507.678387] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:43925 to 0x640000400:43969 [718507.678398] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:43918 to 0x980000402:43937 [718507.741726] Lustre: Skipped 11 previous similar messages [718508.169469] LustreError: 72076:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff92644bf18050 x1623671563137632/t0(0) o4->f9afcafe-b7fa-076a-0d87-fc0f78fd02d0@10.8.6.20@o2ib6:277/0 lens 488/448 e 0 to 0 dl 1548496217 ref 1 fl Interpret:/2/0 rc 0/0 [718508.194600] Lustre: fir-OST001f: Bulk IO write error with f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6), client will retry: rc = -110 [718557.133962] Lustre: fir-OST001f: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [718557.134195] Lustre: fir-OST0019: deleting orphan objects from 0x0:8496 to 0x0:8513 [718557.134768] Lustre: fir-OST0021: deleting orphan objects from 0x0:8497 to 0x0:8513 [718557.135131] Lustre: fir-OST001d: deleting orphan objects from 0x0:8493 to 0x0:8513 [718557.135224] Lustre: fir-OST001b: deleting orphan objects from 0x0:8493 to 0x0:8513 [718557.135343] Lustre: fir-OST0023: deleting orphan objects from 0x0:8493 to 0x0:8513 [718557.181322] Lustre: Skipped 5 previous similar messages [718557.186781] Lustre: fir-OST001f: deleting orphan objects from 0x0:8498 to 0x0:8513 [720367.966192] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [720367.976457] Lustre: Skipped 45 previous similar messages [720576.022491] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [720951.750337] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721191.425568] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721368.338960] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721553.674248] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721553.684514] Lustre: Skipped 2 previous similar messages [721917.570725] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [721917.580985] Lustre: Skipped 3 previous similar messages [722518.326124] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [722518.336388] Lustre: Skipped 5 previous similar messages [723174.927600] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [723174.937865] Lustre: Skipped 7 previous similar messages [725788.754463] Lustre: fir-OST0021: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [725788.764725] Lustre: Skipped 1 previous similar message [726989.968395] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [727195.715712] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [739464.255319] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [740253.788108] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [744477.889126] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [744761.272837] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [745278.582458] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [745453.606634] Lustre: fir-OST0023: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [745735.737502] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [745803.634660] Lustre: fir-OST0021: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [745826.910913] Lustre: fir-OST001f: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [746006.596034] Lustre: fir-OST0021: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [747058.095283] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747100.453426] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747174.109584] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747174.119857] Lustre: Skipped 5 previous similar messages [747252.838649] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [747252.848917] Lustre: Skipped 1 previous similar message [747520.191935] Lustre: fir-OST0021: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [747520.202288] Lustre: Skipped 3 previous similar messages [747674.365304] Lustre: fir-OST0023: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [748535.785549] Lustre: DEBUG MARKER: Sat Jan 26 10:10:20 2019 [748753.220062] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [748753.227290] Lustre: Skipped 3 previous similar messages [748842.029005] Lustre: fir-OST001f: Connection restored to (at 10.8.9.6@o2ib6) [748842.036153] Lustre: Skipped 5 previous similar messages [749083.622144] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [749083.632668] Lustre: Skipped 5 previous similar messages [751566.857437] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [751566.867962] Lustre: Skipped 5 previous similar messages [752398.317459] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [752398.327985] Lustre: Skipped 2 previous similar messages [752626.040683] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [752626.051039] Lustre: Skipped 5 previous similar messages [752685.278932] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [753326.249002] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [753326.259532] Lustre: Skipped 5 previous similar messages [753500.556608] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [753702.383040] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [754083.768753] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [754382.968404] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [754507.071800] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [754581.755778] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [754675.223931] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [754833.235271] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [754833.245797] Lustre: Skipped 1 previous similar message [755319.666348] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [755319.676871] Lustre: Skipped 3 previous similar messages [755852.173809] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [755852.181039] Lustre: Skipped 6 previous similar messages [756506.757690] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [756506.764925] Lustre: Skipped 4 previous similar messages [758277.416383] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [758277.426734] Lustre: Skipped 2 previous similar messages [758423.285621] Lustre: fir-OST0023: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [758423.295974] Lustre: Skipped 1 previous similar message [758610.941930] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [758610.952190] Lustre: Skipped 2 previous similar messages [758920.825710] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [758920.836056] Lustre: Skipped 9 previous similar messages [759525.390791] Lustre: fir-OST0021: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [759525.401142] Lustre: Skipped 20 previous similar messages [760148.649361] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [760148.659712] Lustre: Skipped 18 previous similar messages [761216.294203] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [761216.304473] Lustre: Skipped 8 previous similar messages [763610.163291] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [763610.173816] Lustre: Skipped 1 previous similar message [763838.261752] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [764002.414377] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [764002.424904] Lustre: Skipped 1 previous similar message [764315.414993] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [764315.425529] Lustre: Skipped 7 previous similar messages [767565.616890] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [767914.198428] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [768173.121211] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [768173.131475] Lustre: Skipped 1 previous similar message [769250.671304] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [769296.027444] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [769296.037991] Lustre: Skipped 2 previous similar messages [769957.324047] Lustre: fir-OST0023: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [771099.896396] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [771347.746379] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [771506.887967] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [772004.818535] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [772004.829055] Lustre: Skipped 1 previous similar message [773155.681044] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [773483.029356] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [776305.939928] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [776370.601339] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [777640.406613] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [778210.763011] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [778276.720587] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [778457.308197] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [778955.264365] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [779289.559284] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [779634.441697] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [779932.966779] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [780111.186280] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [780111.196544] Lustre: Skipped 2 previous similar messages [780531.711780] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [780531.722047] Lustre: Skipped 1 previous similar message [781322.641755] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [781322.652021] Lustre: Skipped 3 previous similar messages [781968.960315] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [781968.970576] Lustre: Skipped 1 previous similar message [782676.825907] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [782676.836171] Lustre: Skipped 6 previous similar messages [783451.800403] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [783451.810697] Lustre: Skipped 6 previous similar messages [784174.980942] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [784174.991209] Lustre: Skipped 2 previous similar messages [789066.656332] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [789066.666602] Lustre: Skipped 1 previous similar message [789447.638436] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [790593.481592] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [790744.630452] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [791255.493875] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [791255.504139] Lustre: Skipped 1 previous similar message [791397.329017] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [791672.665498] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [792255.271679] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [792255.281942] Lustre: Skipped 1 previous similar message [793090.413236] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [793090.423499] Lustre: Skipped 2 previous similar messages [794028.497629] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [794028.508153] Lustre: Skipped 5 previous similar messages [795063.256435] Lustre: fir-OST0021: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [795063.266793] Lustre: Skipped 16 previous similar messages [795936.386838] Lustre: fir-OST0021: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [795936.397206] Lustre: Skipped 7 previous similar messages [797107.180871] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [797107.191138] Lustre: Skipped 3 previous similar messages [801919.080888] md: data-check of RAID array md1 [801925.159322] md: data-check of RAID array md7 [801931.220710] md: data-check of RAID array md9 [801937.258199] md: data-check of RAID array md3 [801943.292370] md: data-check of RAID array md11 [801949.360599] md: data-check of RAID array md5 [802166.495314] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802305.488743] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802305.499284] Lustre: Skipped 1 previous similar message [802530.929910] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802530.940457] Lustre: Skipped 5 previous similar messages [802850.436102] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [802850.446626] Lustre: Skipped 5 previous similar messages [805988.851337] Lustre: fir-OST001b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [805988.861707] Lustre: Skipped 3 previous similar messages [809470.632847] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [809470.643364] Lustre: Skipped 2 previous similar messages [809915.161166] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [810840.790485] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [810840.801006] Lustre: Skipped 1 previous similar message [811529.351433] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [811950.658130] Lustre: fir-OST001b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [812548.146430] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [812630.264820] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [812742.475531] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [813149.385452] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [813243.134012] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814211.624651] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814511.744420] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [814800.293660] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815019.578981] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815257.358623] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815423.662983] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815423.673526] Lustre: Skipped 2 previous similar messages [815581.280424] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815581.290952] Lustre: Skipped 2 previous similar messages [815928.684532] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [815928.695058] Lustre: Skipped 5 previous similar messages [816536.916798] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [816536.927318] Lustre: Skipped 5 previous similar messages [817151.676639] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [817151.687182] Lustre: Skipped 7 previous similar messages [817870.967316] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [817870.977840] Lustre: Skipped 8 previous similar messages [818522.665904] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [818522.676424] Lustre: Skipped 13 previous similar messages [819353.783724] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [819353.793985] Lustre: Skipped 1 previous similar message [821675.687881] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [825081.942935] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [825081.953201] Lustre: Skipped 5 previous similar messages [826437.264512] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [826437.274773] Lustre: Skipped 1 previous similar message [826672.486914] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [826672.497174] Lustre: Skipped 2 previous similar messages [826899.634481] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827423.886338] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827575.732460] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827575.742724] Lustre: Skipped 1 previous similar message [827810.357451] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [827967.952473] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [828113.657429] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [828617.379736] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [829278.510068] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [829966.287016] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [829966.297294] Lustre: Skipped 3 previous similar messages [830614.397156] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [831247.896426] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [831247.906687] Lustre: Skipped 3 previous similar messages [835076.038219] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [835076.048483] Lustre: Skipped 4 previous similar messages [835163.056806] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [835408.468472] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [835408.478732] Lustre: Skipped 1 previous similar message [835822.769482] Lustre: fir-OST001d: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [835822.779830] Lustre: Skipped 4 previous similar messages [836774.517935] Lustre: fir-OST0023: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [836774.528289] Lustre: Skipped 8 previous similar messages [837390.243287] Lustre: fir-OST001d: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [837390.253634] Lustre: Skipped 41 previous similar messages [838005.158115] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [838005.168468] Lustre: Skipped 30 previous similar messages [838622.006025] Lustre: fir-OST0019: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [838622.016381] Lustre: Skipped 23 previous similar messages [839240.834306] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [839240.844653] Lustre: Skipped 32 previous similar messages [839933.744713] Lustre: fir-OST0023: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [839933.755069] Lustre: Skipped 27 previous similar messages [840625.413371] Lustre: fir-OST001f: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [840625.423638] Lustre: Skipped 26 previous similar messages [841225.849882] Lustre: fir-OST001f: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [841225.860227] Lustre: Skipped 27 previous similar messages [841830.254495] Lustre: fir-OST0021: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [841830.264851] Lustre: Skipped 12 previous similar messages [842540.129567] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [842540.139919] Lustre: Skipped 18 previous similar messages [843372.554168] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [843372.564540] Lustre: Skipped 8 previous similar messages [843997.583462] Lustre: fir-OST001f: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [843997.593811] Lustre: Skipped 6 previous similar messages [845889.709336] Lustre: fir-OST0021: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [845889.719680] Lustre: Skipped 1 previous similar message [846060.123334] Lustre: fir-OST0023: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [846471.419747] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [846778.086512] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [846778.097142] Lustre: Skipped 18 previous similar messages [847382.452225] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [847382.462744] Lustre: Skipped 37 previous similar messages [847992.956178] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [847992.966707] Lustre: Skipped 39 previous similar messages [848673.451210] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [848673.461741] Lustre: Skipped 30 previous similar messages [849298.224212] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [849298.234734] Lustre: Skipped 34 previous similar messages [849913.627396] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [849913.637926] Lustre: Skipped 41 previous similar messages [850578.301752] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [850578.312282] Lustre: Skipped 14 previous similar messages [851232.154103] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [851232.164630] Lustre: Skipped 10 previous similar messages [851839.334159] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [851839.344696] Lustre: Skipped 21 previous similar messages [852443.835147] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [852443.845672] Lustre: Skipped 24 previous similar messages [853044.728179] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [853044.738702] Lustre: Skipped 16 previous similar messages [853808.197429] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [853808.207962] Lustre: Skipped 14 previous similar messages [854419.550856] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [854419.561401] Lustre: Skipped 9 previous similar messages [855028.496427] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [855028.506958] Lustre: Skipped 15 previous similar messages [855756.870637] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [855756.880904] Lustre: Skipped 6 previous similar messages [856735.011948] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [856735.022242] Lustre: Skipped 1 previous similar message [857457.152681] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [857457.159923] Lustre: Skipped 7 previous similar messages [858057.182410] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [858057.189638] Lustre: Skipped 13 previous similar messages [858957.264286] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [858957.271516] Lustre: Skipped 19 previous similar messages [859557.315499] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [859557.322777] Lustre: Skipped 11 previous similar messages [860158.281303] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [860158.288531] Lustre: Skipped 13 previous similar messages [861058.321544] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [861058.328770] Lustre: Skipped 18 previous similar messages [861957.353321] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [861957.360551] Lustre: Skipped 17 previous similar messages [862817.445475] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [862817.455740] Lustre: Skipped 19 previous similar messages [863543.128910] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [864291.361808] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [865092.446360] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [865092.456627] Lustre: Skipped 2 previous similar messages [866052.197094] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [866052.207363] Lustre: Skipped 3 previous similar messages [867211.994465] Lustre: fir-OST001b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [867212.004847] Lustre: Skipped 14 previous similar messages [867888.030173] Lustre: fir-OST001d: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [867888.040549] Lustre: Skipped 4 previous similar messages [868764.883127] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [868764.893390] Lustre: Skipped 3 previous similar messages [869367.115020] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [869367.125284] Lustre: Skipped 5 previous similar messages [870107.004288] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [870107.014633] Lustre: Skipped 4 previous similar messages [870764.622095] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [870764.632359] Lustre: Skipped 5 previous similar messages [871407.610105] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [871407.620373] Lustre: Skipped 7 previous similar messages [872020.746887] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [872020.757162] Lustre: Skipped 4 previous similar messages [872642.958604] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [872642.968874] Lustre: Skipped 14 previous similar messages [873272.802404] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [873272.812667] Lustre: Skipped 8 previous similar messages [874004.071117] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [874004.081469] Lustre: Skipped 9 previous similar messages [875075.230015] Lustre: fir-OST0021: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [875075.240362] Lustre: Skipped 1 previous similar message [875767.943811] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [875767.954160] Lustre: Skipped 12 previous similar messages [876522.924599] Lustre: fir-OST0019: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [876522.934953] Lustre: Skipped 1 previous similar message [877129.697600] Lustre: fir-OST001d: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [877129.707866] Lustre: Skipped 26 previous similar messages [878442.555269] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [878442.565799] Lustre: Skipped 8 previous similar messages [878653.226227] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [879352.776462] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [879352.786813] Lustre: Skipped 5 previous similar messages [879862.233576] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [881180.021755] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [881180.032286] Lustre: Skipped 6 previous similar messages [881970.263558] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [881970.273912] Lustre: Skipped 5 previous similar messages [883209.275918] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [883677.485257] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [883677.495602] Lustre: Skipped 5 previous similar messages [884545.307352] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [884545.317701] Lustre: Skipped 5 previous similar messages [885775.849507] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [885955.323648] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [886187.049584] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [886202.808125] Lustre: fir-OST0021: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [886235.767696] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [886235.778045] Lustre: Skipped 5 previous similar messages [886312.351626] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [886507.619046] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [887386.606694] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [887737.492921] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [887914.890181] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [887914.900530] Lustre: Skipped 5 previous similar messages [888290.732940] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [888778.846057] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [888778.856407] Lustre: Skipped 5 previous similar messages [889230.564861] Lustre: fir-OST0023: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [889520.528252] Lustre: fir-OST0021: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [889520.538601] Lustre: Skipped 1 previous similar message [889925.762704] Lustre: fir-OST0019: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [889925.772994] Lustre: Skipped 1 previous similar message [891275.669345] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [891275.669345] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [891275.669349] Lustre: Skipped 9 previous similar messages [891275.695365] Lustre: Skipped 4 previous similar messages [892255.296073] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [892922.961424] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [892922.971778] Lustre: Skipped 5 previous similar messages [893619.339157] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [893806.299133] Lustre: fir-OST0021: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [894588.183935] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [894588.194286] Lustre: Skipped 5 previous similar messages [896288.797501] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [896288.807859] Lustre: Skipped 3 previous similar messages [898003.802560] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [898003.812923] Lustre: Skipped 5 previous similar messages [898962.274486] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [899720.493390] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [899720.503747] Lustre: Skipped 5 previous similar messages [901417.117582] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [901417.127940] Lustre: Skipped 5 previous similar messages [901504.254231] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [901934.326004] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [902025.368292] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [902025.375527] Lustre: Skipped 5 previous similar messages [902226.072738] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [902226.079972] Lustre: Skipped 5 previous similar messages [902319.565906] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [902319.573131] Lustre: Skipped 5 previous similar messages [902404.106235] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [902404.113469] Lustre: Skipped 5 previous similar messages [902542.675468] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [902542.685750] Lustre: Skipped 5 previous similar messages [902949.836406] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [903129.197715] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [903129.208080] Lustre: Skipped 5 previous similar messages [903603.098135] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [903603.108401] Lustre: Skipped 2 previous similar messages [904812.284101] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [904812.294457] Lustre: Skipped 6 previous similar messages [906002.720059] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [906473.872179] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [906473.882532] Lustre: Skipped 5 previous similar messages [908197.732369] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [908197.742719] Lustre: Skipped 5 previous similar messages [909087.175417] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [909087.185771] Lustre: Skipped 5 previous similar messages [909960.335152] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [909960.345507] Lustre: Skipped 5 previous similar messages [911473.434891] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911525.760450] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911575.782118] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911636.738586] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [911636.748937] Lustre: Skipped 5 previous similar messages [911664.297176] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911755.394535] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911842.789152] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [911896.776067] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [912026.565875] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [912026.576396] Lustre: Skipped 1 previous similar message [912238.109790] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [912578.268541] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [912578.279065] Lustre: Skipped 8 previous similar messages [913097.437206] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [913097.447733] Lustre: Skipped 9 previous similar messages [913715.654094] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [913715.664615] Lustre: Skipped 5 previous similar messages [914335.100882] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [914335.111412] Lustre: Skipped 13 previous similar messages [914939.307957] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [914939.318481] Lustre: Skipped 23 previous similar messages [915898.673301] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [915898.683648] Lustre: Skipped 11 previous similar messages [917549.039877] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [917549.039878] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [917549.060660] Lustre: Skipped 3 previous similar messages [917658.062233] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [917658.072761] Lustre: Skipped 1 previous similar message [917991.446799] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [917991.457321] Lustre: Skipped 6 previous similar messages [918306.690761] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [918306.701288] Lustre: Skipped 7 previous similar messages [919230.303271] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [919230.313626] Lustre: Skipped 12 previous similar messages [920050.821371] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [920050.831719] Lustre: Skipped 6 previous similar messages [921076.892319] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [921692.103417] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [921692.113777] Lustre: Skipped 5 previous similar messages [922591.694648] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [923430.634003] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [923430.644356] Lustre: Skipped 5 previous similar messages [924365.959064] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [925353.693864] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [925353.704219] Lustre: Skipped 5 previous similar messages [925964.186624] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [925964.197150] Lustre: Skipped 4 previous similar messages [926899.400798] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [926899.411345] Lustre: Skipped 18 previous similar messages [927604.659077] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [927604.669343] Lustre: Skipped 8 previous similar messages [928896.593122] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [928896.603471] Lustre: Skipped 6 previous similar messages [930593.972097] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [930593.982447] Lustre: Skipped 5 previous similar messages [932308.051838] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [932308.062192] Lustre: Skipped 5 previous similar messages [933097.373075] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [933281.034152] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [933844.377189] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [933935.081942] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [934012.272285] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [934012.282642] Lustre: Skipped 5 previous similar messages [934134.008912] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [934180.624728] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [934304.793176] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [934304.803698] Lustre: Skipped 1 previous similar message [934627.408463] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [935079.473219] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [935605.695527] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [935605.706075] Lustre: Skipped 1 previous similar message [936160.903481] Lustre: fir-OST0023: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [936160.914005] Lustre: Skipped 8 previous similar messages [936782.221810] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [936782.232450] Lustre: Skipped 9 previous similar messages [937655.307374] Lustre: fir-OST0021: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [937655.317637] Lustre: Skipped 1 previous similar message [938423.791833] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [938423.802184] Lustre: Skipped 6 previous similar messages [939367.799007] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [939611.084763] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [939611.095023] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.3@o2ib7 (107): c: 8, oc: 0, rc: 8 [940286.308552] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [940286.318900] Lustre: Skipped 6 previous similar messages [941161.790931] Lustre: fir-OST0023: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [941161.801282] Lustre: Skipped 10 previous similar messages [942425.866010] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [942425.876363] Lustre: Skipped 13 previous similar messages [942553.278544] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [942553.288891] Lustre: Skipped 1 previous similar message [943375.008145] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [943375.015385] Lustre: Skipped 1 previous similar message [943470.833041] Lustre: fir-OST001f: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [943470.843396] Lustre: Skipped 5 previous similar messages [943522.799670] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [943634.201090] Lustre: fir-OST0023: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [943634.211447] Lustre: Skipped 2 previous similar messages [943815.560583] Lustre: fir-OST0019: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [943815.570854] Lustre: Skipped 9 previous similar messages [944169.436344] Lustre: fir-OST0023: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [944169.446694] Lustre: Skipped 16 previous similar messages [944786.402844] Lustre: fir-OST001f: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [944786.413195] Lustre: Skipped 33 previous similar messages [945408.803531] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [945408.813883] Lustre: Skipped 30 previous similar messages [946067.354603] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [946067.365131] Lustre: Skipped 17 previous similar messages [946675.097607] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [946675.104831] Lustre: Skipped 13 previous similar messages [947276.113588] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [947276.120832] Lustre: Skipped 44 previous similar messages [947887.831060] Lustre: fir-OST001f: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [947887.841415] Lustre: Skipped 73 previous similar messages [948489.800030] Lustre: fir-OST001b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [948489.810295] Lustre: Skipped 51 previous similar messages [949102.980492] Lustre: fir-OST001f: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [949102.990847] Lustre: Skipped 59 previous similar messages [949709.823848] Lustre: fir-OST001f: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [949709.834220] Lustre: Skipped 61 previous similar messages [950575.270557] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [950575.277789] Lustre: Skipped 18 previous similar messages [951475.207551] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [951475.214852] Lustre: Skipped 17 previous similar messages [952156.407240] Lustre: fir-OST0019: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [952156.417614] Lustre: Skipped 28 previous similar messages [952975.206701] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [952975.213928] Lustre: Skipped 25 previous similar messages [953575.222652] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [953575.229942] Lustre: Skipped 11 previous similar messages [954175.264264] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [954175.271488] Lustre: Skipped 11 previous similar messages [954775.281845] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [954775.289069] Lustre: Skipped 16 previous similar messages [955376.341390] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [955376.348633] Lustre: Skipped 6 previous similar messages [955976.359250] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [955976.366523] Lustre: Skipped 14 previous similar messages [956576.376576] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [956576.383804] Lustre: Skipped 13 previous similar messages [957476.396243] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [957476.403472] Lustre: Skipped 12 previous similar messages [958375.450614] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [958375.457854] Lustre: Skipped 17 previous similar messages [959275.385032] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [959275.392255] Lustre: Skipped 17 previous similar messages [959876.401449] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [959876.408785] Lustre: Skipped 11 previous similar messages [960775.427051] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [960775.434278] Lustre: Skipped 17 previous similar messages [961375.444476] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [961375.451708] Lustre: Skipped 16 previous similar messages [961930.713102] Lustre: fir-OST001d: haven't heard from client 9d0731b3-b468-e234-aad6-42e89a84a6d4 (at 10.9.101.60@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92839d53bc00, cur 1548739610 expire 1548739460 last 1548739383 [962275.464837] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [962275.472086] Lustre: Skipped 12 previous similar messages [962875.482268] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [962875.489514] Lustre: Skipped 13 previous similar messages [963475.501376] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [963475.508602] Lustre: Skipped 10 previous similar messages [964076.512961] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [964076.520192] Lustre: Skipped 11 previous similar messages [964975.539241] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [964975.546471] Lustre: Skipped 17 previous similar messages [965575.557456] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [965575.564696] Lustre: Skipped 14 previous similar messages [966227.471824] Lustre: fir-OST0019: Connection restored to (at 10.9.101.60@o2ib4) [966227.479227] Lustre: Skipped 19 previous similar messages [967075.656540] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [967075.663769] Lustre: Skipped 18 previous similar messages [967676.671719] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [967676.678945] Lustre: Skipped 11 previous similar messages [968575.696192] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [968575.703426] Lustre: Skipped 17 previous similar messages [969175.714275] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [969175.721573] Lustre: Skipped 11 previous similar messages [969832.683057] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [969832.693322] Lustre: Skipped 21 previous similar messages [970675.709511] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [970675.716733] Lustre: Skipped 12 previous similar messages [971576.776358] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [971576.783591] Lustre: Skipped 17 previous similar messages [972475.803487] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [972475.810712] Lustre: Skipped 17 previous similar messages [973076.857069] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [973076.864528] Lustre: Skipped 11 previous similar messages [973976.845441] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [973976.852671] Lustre: Skipped 17 previous similar messages [974875.864978] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [974875.872261] Lustre: Skipped 17 previous similar messages [975475.883942] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [975475.891173] Lustre: Skipped 16 previous similar messages [976376.845231] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [976376.852468] Lustre: Skipped 12 previous similar messages [977275.931216] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [977275.938465] Lustre: Skipped 17 previous similar messages [978175.953258] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [978175.960599] Lustre: Skipped 17 previous similar messages [978775.971758] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [978775.979022] Lustre: Skipped 15 previous similar messages [979375.989272] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [979375.996501] Lustre: Skipped 12 previous similar messages [980275.992902] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [980276.000135] Lustre: Skipped 12 previous similar messages [980876.010188] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [980876.017413] Lustre: Skipped 17 previous similar messages [981477.024426] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [981477.031654] Lustre: Skipped 10 previous similar messages [982377.048084] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [982377.055335] Lustre: Skipped 19 previous similar messages [982977.072243] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [982977.079477] Lustre: Skipped 11 previous similar messages [983631.922257] Lustre: fir-OST0019: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [983631.932526] Lustre: Skipped 17 previous similar messages [984476.102290] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [984476.109514] Lustre: Skipped 15 previous similar messages [985302.642256] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [985302.652519] Lustre: Skipped 21 previous similar messages [985976.102019] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [985976.109280] Lustre: Skipped 14 previous similar messages [986747.087921] Lustre: fir-OST001b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [986747.098188] Lustre: Skipped 21 previous similar messages [987364.202787] Lustre: fir-OST0023: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [987364.213146] Lustre: Skipped 17 previous similar messages [988076.418945] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [988076.426178] Lustre: Skipped 15 previous similar messages [988977.218697] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [988977.225922] Lustre: Skipped 17 previous similar messages [989577.236736] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [989577.243967] Lustre: Skipped 15 previous similar messages [990354.296090] Lustre: fir-OST0021: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [990354.306372] Lustre: Skipped 16 previous similar messages [991076.275521] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [991076.282754] Lustre: Skipped 15 previous similar messages [991976.255984] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [991976.263219] Lustre: Skipped 17 previous similar messages [992576.292200] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [992576.299426] Lustre: Skipped 14 previous similar messages [993176.966617] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [993176.973844] Lustre: Skipped 8 previous similar messages [994076.368139] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [994076.375371] Lustre: Skipped 17 previous similar messages [994676.386126] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [994676.393370] Lustre: Skipped 16 previous similar messages [995276.404072] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [995276.411296] Lustre: Skipped 10 previous similar messages [995877.415561] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [995877.422788] Lustre: Skipped 7 previous similar messages [996776.438827] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [996776.446112] Lustre: Skipped 17 previous similar messages [997377.453944] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [997377.461190] Lustre: Skipped 11 previous similar messages [997977.472033] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [997977.479284] Lustre: Skipped 11 previous similar messages [998877.492963] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [998877.500195] Lustre: Skipped 17 previous similar messages [999776.517508] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [999776.525065] Lustre: Skipped 17 previous similar messages [1000377.536986] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1000377.544306] Lustre: Skipped 11 previous similar messages [1001276.560187] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1001276.567520] Lustre: Skipped 17 previous similar messages [1002176.520289] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1002176.527616] Lustre: Skipped 17 previous similar messages [1002776.537884] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1002776.545209] Lustre: Skipped 14 previous similar messages [1003376.555746] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1003376.563060] Lustre: Skipped 15 previous similar messages [1003977.597262] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1003977.604630] Lustre: Skipped 9 previous similar messages [1004752.822086] Lustre: fir-OST001b: Connection restored to 45d9bd1b-c260-5ee3-c014-76ee236dd98f (at 10.8.6.31@o2ib6) [1004752.832523] Lustre: Skipped 22 previous similar messages [1005380.412063] Lustre: fir-OST001f: Connection restored to (at 10.8.9.6@o2ib6) [1005380.419296] Lustre: Skipped 13 previous similar messages [1006076.608477] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1006076.615793] Lustre: Skipped 14 previous similar messages [1006676.624261] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1006676.631636] Lustre: Skipped 37 previous similar messages [1007276.639607] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1007276.646926] Lustre: Skipped 24 previous similar messages [1007876.655065] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1007876.662374] Lustre: Skipped 34 previous similar messages [1008476.702941] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1008476.710283] Lustre: Skipped 21 previous similar messages [1009076.719696] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1009076.727031] Lustre: Skipped 20 previous similar messages [1009694.011334] Lustre: fir-OST0023: Connection restored to 75d3df1c-c05f-80f2-4e0d-3991a06b4cf2 (at 10.8.2.9@o2ib6) [1009694.021685] Lustre: Skipped 19 previous similar messages [1010338.199243] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1010338.209851] Lustre: Skipped 27 previous similar messages [1010998.236308] Lustre: fir-OST0021: Connection restored to (at 10.8.9.6@o2ib6) [1010998.243537] Lustre: Skipped 17 previous similar messages [1011602.649915] Lustre: fir-OST001f: Connection restored to (at 10.8.9.6@o2ib6) [1011602.657162] Lustre: Skipped 18 previous similar messages [1012148.805013] Lustre: fir-OST001b: Client a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2 (at 10.9.101.60@o2ib4) reconnecting [1012148.815361] Lustre: Skipped 1 previous similar message [1012149.453688] LustreError: 72185:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff926d6ba01850 x1623976115400992/t0(0) o4->a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2@10.9.101.60@o2ib4:228/0 lens 488/448 e 0 to 0 dl 1548789863 ref 1 fl Interpret:/0/0 rc 0/0 [1012149.805041] Lustre: fir-OST001d: Client a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2 (at 10.9.101.60@o2ib4) reconnecting [1012149.815393] Lustre: Skipped 1 previous similar message [1012152.759759] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1012152.769930] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.209@o2ib7 (2): c: 0, oc: 1, rc: 8 [1012153.411786] Lustre: fir-OST0023: Client 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) reconnecting [1012153.422129] Lustre: Skipped 2 previous similar messages [1012153.716795] LustreError: 64332:0:(ldlm_lib.c:3258:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff928d8c674050 x1623602976162144/t0(0) o4->28764730-adc8-1b1a-0aa7-9bb3363d4889@10.9.101.59@o2ib4:230/0 lens 488/448 e 0 to 0 dl 1548789865 ref 1 fl Interpret:/0/0 rc 0/0 [1012155.176092] LNetError: 61898:0:(lib-msg.c:811:lnet_is_health_check()) Msg is in inconsistent state, don't perform health checking (-125, 0) [1012155.188782] LustreError: 61898:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928fe50f5000 [1012155.199760] Lustre: fir-OST0021: Bulk IO write error with a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2 (at 10.9.101.60@o2ib4), client will retry: rc = -110 [1012155.280977] LustreError: 61897:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012155.291937] LustreError: 61898:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012155.302896] LustreError: 61899:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012156.351991] LNetError: 61899:0:(lib-msg.c:811:lnet_is_health_check()) Msg is in inconsistent state, don't perform health checking (-125, 0) [1012156.364684] LNetError: 61899:0:(lib-msg.c:811:lnet_is_health_check()) Skipped 3 previous similar messages [1012156.374435] LustreError: 61899:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.385400] LustreError: 61899:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.396361] LustreError: 61896:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.407343] LustreError: 61898:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.418302] LustreError: 61899:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.429260] LustreError: 61896:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.440219] LustreError: 61898:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.451182] LustreError: 61899:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff929153d1ca00 [1012156.462579] Lustre: fir-OST0023: Bulk IO write error with 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4), client will retry: rc = -110 [1012158.446937] LNetError: 61891:0:(lib-msg.c:811:lnet_is_health_check()) Msg is in inconsistent state, don't perform health checking (-125, 0) [1012158.446942] LustreError: 61889:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012158.446945] LustreError: 61890:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012158.481657] LNetError: 61891:0:(lib-msg.c:811:lnet_is_health_check()) Skipped 9 previous similar messages [1012158.491781] LustreError: 61891:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012158.638037] LustreError: 61902:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012158.648997] LustreError: 61901:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012158.659949] LustreError: 61900:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012158.670915] LustreError: 61902:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012163.481802] LNetError: 61892:0:(lib-msg.c:811:lnet_is_health_check()) Msg is in inconsistent state, don't perform health checking (-125, 0) [1012163.494387] LustreError: 61894:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012163.505478] LNetError: 61892:0:(lib-msg.c:811:lnet_is_health_check()) Skipped 5 previous similar messages [1012163.515223] LustreError: 61892:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff928ec1a0c800 [1012163.527457] Lustre: fir-OST0023: Bulk IO write error with 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4), client will retry: rc = -110 [1012170.058906] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012170.076449] LustreError: Skipped 1 previous similar message [1012170.313966] Lustre: fir-OST0019: Client 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) reconnecting [1012170.324054] Lustre: Skipped 7 previous similar messages [1012181.262112] LustreError: 137-5: fir-OST0022_UUID: not available for connect from 10.9.0.2@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012181.279392] LustreError: Skipped 1 previous similar message [1012192.277410] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.0.1@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012192.294691] LustreError: Skipped 4 previous similar messages [1012195.110928] Lustre: fir-OST001b: Client a0fd8789-8ae9-fb75-85b8-13d1c4cef8d2 (at 10.9.101.60@o2ib4) reconnecting [1012195.110996] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.101.60@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1012195.138881] Lustre: Skipped 2 previous similar messages [1012206.262814] Lustre: fir-OST0023: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [1012206.273189] Lustre: Skipped 33 previous similar messages [1012217.475292] Lustre: fir-OST0019: Client 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) reconnecting [1012217.485382] Lustre: Skipped 5 previous similar messages [1012260.184065] Lustre: fir-OST0019: Client 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) reconnecting [1012260.194160] Lustre: Skipped 7 previous similar messages [1012870.062960] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1012870.073579] Lustre: Skipped 38 previous similar messages [1013837.338247] Lustre: fir-OST0023: Connection restored to (at 10.8.9.6@o2ib6) [1013837.345477] Lustre: Skipped 26 previous similar messages [1014499.267032] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1014499.277476] Lustre: Skipped 13 previous similar messages [1015316.552057] Lustre: fir-OST0023: Connection restored to (at 10.8.9.6@o2ib6) [1015316.559291] Lustre: Skipped 12 previous similar messages [1015977.870923] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1015977.878241] Lustre: Skipped 44 previous similar messages [1016587.513256] Lustre: fir-OST001f: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1016587.523725] Lustre: Skipped 44 previous similar messages [1017342.726589] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [1017342.733904] Lustre: Skipped 80 previous similar messages [1018076.980807] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1018076.988148] Lustre: Skipped 57 previous similar messages [1018976.991232] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1018976.998562] Lustre: Skipped 33 previous similar messages [1019578.007460] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1019578.014773] Lustre: Skipped 55 previous similar messages [1020179.354127] Lustre: fir-OST001b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1020179.364472] Lustre: Skipped 59 previous similar messages [1020833.962980] Lustre: fir-OST001b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1020833.973419] Lustre: Skipped 78 previous similar messages [1021475.237040] Lustre: fir-OST001d: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1021475.247471] Lustre: Skipped 38 previous similar messages [1022170.506340] Lustre: fir-OST001f: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1022170.516777] Lustre: Skipped 60 previous similar messages [1022878.103916] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1022878.111242] Lustre: Skipped 65 previous similar messages [1023777.127202] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1023777.134518] Lustre: Skipped 20 previous similar messages [1024377.145399] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1024377.152714] Lustre: Skipped 13 previous similar messages [1024977.163871] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1024977.171204] Lustre: Skipped 9 previous similar messages [1025877.229148] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1025877.236468] Lustre: Skipped 30 previous similar messages [1026518.676819] Lustre: fir-OST001b: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1026518.687250] Lustre: Skipped 19 previous similar messages [1027350.361601] Lustre: fir-OST0021: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1027350.372040] Lustre: Skipped 17 previous similar messages [1027978.178748] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1027978.186081] Lustre: Skipped 27 previous similar messages [1028681.040157] Lustre: fir-OST0021: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1028681.050590] Lustre: Skipped 27 previous similar messages [1029315.640094] Lustre: fir-OST0023: Connection restored to (at 10.9.101.60@o2ib4) [1029315.647586] Lustre: Skipped 17 previous similar messages [1030077.234421] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1030077.241738] Lustre: Skipped 20 previous similar messages [1030677.251199] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1030677.258514] Lustre: Skipped 14 previous similar messages [1031277.267289] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1031277.274605] Lustre: Skipped 14 previous similar messages [1032336.386443] Lustre: fir-OST001b: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1032336.396909] Lustre: Skipped 16 previous similar messages [1033492.160076] Lustre: fir-OST0019: Connection restored to 8ae4cc68-204d-d2bb-4908-94f49692744b (at 10.8.20.15@o2ib6) [1033492.170610] Lustre: Skipped 8 previous similar messages [1034101.819759] Lustre: fir-OST0021: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1038056.879012] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1038056.889429] Lustre: Skipped 1 previous similar message [1038143.191928] Lustre: fir-OST0023: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1038143.202396] Lustre: Skipped 3 previous similar messages [1039131.228210] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1039131.238562] Lustre: Skipped 16 previous similar messages [1039525.705362] Lustre: fir-OST0021: Connection restored to (at 10.9.101.60@o2ib4) [1039525.712852] Lustre: Skipped 3 previous similar messages [1039871.651735] Lustre: fir-OST001d: Connection restored to (at 10.9.101.60@o2ib4) [1039871.659221] Lustre: Skipped 3 previous similar messages [1040109.818777] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1040291.058232] Lustre: fir-OST0021: Connection restored to (at 10.9.101.60@o2ib4) [1040291.065777] Lustre: Skipped 1 previous similar message [1040664.418108] Lustre: fir-OST001f: Connection restored to (at 10.9.101.60@o2ib4) [1040664.425599] Lustre: Skipped 6 previous similar messages [1041548.239463] Lustre: fir-OST001d: Connection restored to (at 10.9.101.60@o2ib4) [1041548.246967] Lustre: Skipped 15 previous similar messages [1042234.700727] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1042234.711108] Lustre: Skipped 24 previous similar messages [1042837.870577] Lustre: fir-OST001b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1042837.881045] Lustre: Skipped 2 previous similar messages [1043549.545517] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1043549.555873] Lustre: Skipped 23 previous similar messages [1044201.229835] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1044201.240187] Lustre: Skipped 11 previous similar messages [1044889.509440] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1044889.519790] Lustre: Skipped 11 previous similar messages [1045494.810079] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1045494.820429] Lustre: Skipped 13 previous similar messages [1046151.588015] Lustre: fir-OST001d: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1046151.598456] Lustre: Skipped 13 previous similar messages [1046939.761497] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1046939.771848] Lustre: Skipped 27 previous similar messages [1047568.588182] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1047568.598527] Lustre: Skipped 9 previous similar messages [1048592.560456] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1048592.570805] Lustre: Skipped 5 previous similar messages [1049243.364472] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1049243.374821] Lustre: Skipped 1 previous similar message [1049945.979837] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1049945.990189] Lustre: Skipped 3 previous similar messages [1050800.323426] Lustre: fir-OST0019: Connection restored to (at 10.9.101.60@o2ib4) [1050800.330944] Lustre: Skipped 3 previous similar messages [1051484.194263] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1051484.204618] Lustre: Skipped 27 previous similar messages [1052104.430481] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1052104.440830] Lustre: Skipped 9 previous similar messages [1052733.584268] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1052733.594622] Lustre: Skipped 16 previous similar messages [1053409.493743] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1053409.504090] Lustre: Skipped 14 previous similar messages [1054288.035866] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1054288.046216] Lustre: Skipped 7 previous similar messages [1055646.662696] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1055646.673048] Lustre: Skipped 5 previous similar messages [1055850.576095] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1055850.586451] Lustre: Skipped 1 previous similar message [1056073.108714] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1056073.119062] Lustre: Skipped 5 previous similar messages [1058086.749834] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1058551.127852] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1058551.138197] Lustre: Skipped 1 previous similar message [1058760.061418] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1058760.071768] Lustre: Skipped 2 previous similar messages [1059005.460758] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1059005.471104] Lustre: Skipped 2 previous similar messages [1059602.988052] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1059602.998404] Lustre: Skipped 1 previous similar message [1060206.489959] Lustre: fir-OST0021: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1060206.500569] Lustre: Skipped 5 previous similar messages [1061454.511518] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1061454.521868] Lustre: Skipped 12 previous similar messages [1061585.930046] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1061937.578189] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1061937.588537] Lustre: Skipped 2 previous similar messages [1062663.678104] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1062663.688495] Lustre: Skipped 3 previous similar messages [1063269.488055] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1063269.498401] Lustre: Skipped 4 previous similar messages [1064080.949688] Lustre: fir-OST001d: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1064080.960065] Lustre: Skipped 5 previous similar messages [1064856.060374] Lustre: fir-OST001b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1064856.070810] Lustre: Skipped 13 previous similar messages [1065485.782780] Lustre: fir-OST001b: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1065485.793223] Lustre: Skipped 20 previous similar messages [1066089.148346] Lustre: fir-OST0023: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1066089.158783] Lustre: Skipped 23 previous similar messages [1066742.645549] Lustre: fir-OST001d: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1066742.655990] Lustre: Skipped 21 previous similar messages [1067349.714469] Lustre: fir-OST0019: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1067349.724817] Lustre: Skipped 7 previous similar messages [1068152.656932] Lustre: fir-OST001b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1068152.667307] Lustre: Skipped 2 previous similar messages [1068988.704421] Lustre: fir-OST001d: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1068988.714808] Lustre: Skipped 3 previous similar messages [1069834.796108] Lustre: fir-OST0019: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1070442.564016] Lustre: fir-OST001b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1070442.574361] Lustre: Skipped 30 previous similar messages [1071055.923663] Lustre: fir-OST0019: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1071055.934101] Lustre: Skipped 21 previous similar messages [1072803.750045] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1072803.760676] Lustre: Skipped 9 previous similar messages [1072989.013537] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1072989.023882] Lustre: Skipped 4 previous similar messages [1073164.537849] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1073164.548208] Lustre: Skipped 1 previous similar message [1073465.176706] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1073465.187113] Lustre: Skipped 2 previous similar messages [1074199.852895] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1074199.863270] Lustre: Skipped 2 previous similar messages [1075170.043224] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1075170.053835] Lustre: Skipped 8 previous similar messages [1075929.697282] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1075929.707629] Lustre: Skipped 20 previous similar messages [1076610.759649] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1076610.770096] Lustre: Skipped 9 previous similar messages [1077274.639577] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1077274.649926] Lustre: Skipped 6 previous similar messages [1080862.548742] Lustre: fir-OST0023: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1080862.559112] Lustre: Skipped 8 previous similar messages [1080946.133050] Lustre: fir-OST001d: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1080946.143485] Lustre: Skipped 4 previous similar messages [1081104.876144] Lustre: fir-OST001d: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1081104.886589] Lustre: Skipped 21 previous similar messages [1081429.237723] Lustre: fir-OST0023: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1081429.248169] Lustre: Skipped 1 previous similar message [1082235.277665] Lustre: fir-OST0019: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1082235.288012] Lustre: Skipped 46 previous similar messages [1082837.482171] Lustre: fir-OST0021: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1082837.492608] Lustre: Skipped 32 previous similar messages [1083439.757570] Lustre: fir-OST001f: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1083439.768010] Lustre: Skipped 34 previous similar messages [1084048.850289] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1084048.860895] Lustre: Skipped 30 previous similar messages [1084669.589563] Lustre: fir-OST001d: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1084669.600179] Lustre: Skipped 20 previous similar messages [1085285.798784] Lustre: fir-OST0023: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1085285.809229] Lustre: Skipped 22 previous similar messages [1085895.693868] Lustre: fir-OST0019: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1085895.704301] Lustre: Skipped 19 previous similar messages [1086497.778211] Lustre: fir-OST001d: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1086497.788568] Lustre: Skipped 25 previous similar messages [1087147.185141] Lustre: fir-OST001f: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1087147.195752] Lustre: Skipped 12 previous similar messages [1087754.140144] Lustre: fir-OST001b: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1087754.150758] Lustre: Skipped 6 previous similar messages [1088396.435144] Lustre: fir-OST001d: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1088396.445584] Lustre: Skipped 21 previous similar messages [1089017.745920] Lustre: fir-OST0019: Connection restored to f9afcafe-b7fa-076a-0d87-fc0f78fd02d0 (at 10.8.6.20@o2ib6) [1089017.756361] Lustre: Skipped 36 previous similar messages [1095461.234307] perf: interrupt took too long (3977 > 3921), lowering kernel.perf_event_max_sample_rate to 50000 [1096113.490732] Lustre: fir-OST001b: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1096113.501089] Lustre: Skipped 29 previous similar messages [1098645.491966] Lustre: fir-OST001b: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1098645.502422] Lustre: Skipped 15 previous similar messages [1098655.424991] Lustre: fir-OST001f: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1098676.072621] Lustre: fir-OST001d: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1098720.278533] Lustre: fir-OST0019: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1098720.288888] Lustre: Skipped 2 previous similar messages [1098849.873350] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1098849.883706] Lustre: Skipped 5 previous similar messages [1099139.684087] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1099139.694441] Lustre: Skipped 7 previous similar messages [1099460.176950] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1099460.187373] Lustre: Skipped 11 previous similar messages [1100182.478938] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1100182.489294] Lustre: Skipped 19 previous similar messages [1100866.337256] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1100866.347603] Lustre: Skipped 11 previous similar messages [1101631.187465] Lustre: fir-OST0021: Connection restored to 9daa8e10-c154-ca96-d85d-3f6355b9ec3f (at 10.8.2.5@o2ib6) [1101631.197814] Lustre: Skipped 10 previous similar messages [1106673.547034] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [1106673.554347] Lustre: Skipped 11 previous similar messages [1119760.107292] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1119760.117647] Lustre: Skipped 6 previous similar messages [1119886.257596] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [1119886.264940] Lustre: Skipped 2 previous similar messages [1121321.016718] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [1121321.024064] Lustre: Skipped 4 previous similar messages [1142133.653174] md: md11: data-check done. [1154313.006629] md: md3: data-check done. [1157960.356753] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [1157961.634377] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [1158418.558660] md: md1: data-check done. [1162296.813631] md: md5: data-check done. [1174776.502361] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [1174776.509684] Lustre: Skipped 1 previous similar message [1174777.737181] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [1174798.534977] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [1178964.906222] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1178964.916575] Lustre: Skipped 1 previous similar message [1179094.070684] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [1179094.077999] Lustre: Skipped 5 previous similar messages [1179102.049664] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [1179102.056980] Lustre: Skipped 1 previous similar message [1179106.223155] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [1179126.193005] Lustre: fir-OST001f: Connection restored to (at 10.8.6.33@o2ib6) [1179509.845906] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [1179509.853219] Lustre: Skipped 1 previous similar message [1180584.342356] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [1180584.349670] Lustre: Skipped 4 previous similar messages [1180600.738092] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [1180600.745408] Lustre: Skipped 2 previous similar messages [1180755.792472] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1180755.802826] Lustre: Skipped 4 previous similar messages [1181293.988314] Lustre: fir-OST001b: Connection restored to (at 10.8.6.33@o2ib6) [1184018.543755] Lustre: fir-OST0021: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1184025.791228] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1184136.850789] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1186220.186252] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [1186220.193570] Lustre: Skipped 7 previous similar messages [1186277.516608] Lustre: fir-OST0023: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1186747.569685] Lustre: fir-OST0023: Connection restored to (at 10.8.6.33@o2ib6) [1186747.576997] Lustre: Skipped 4 previous similar messages [1186918.901088] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1186918.911571] Lustre: Skipped 1 previous similar message [1187196.254412] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187196.264875] Lustre: Skipped 1 previous similar message [1187407.581673] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187407.592152] Lustre: Skipped 1 previous similar message [1187628.912442] Lustre: fir-OST001d: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1187628.922903] Lustre: Skipped 1 previous similar message [1188038.899599] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [1188312.997600] Lustre: fir-OST001d: Connection restored to (at 10.8.6.33@o2ib6) [1188986.850634] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [1191182.500149] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1191182.507466] Lustre: Skipped 19 previous similar messages [1191254.625114] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1191254.632435] Lustre: Skipped 1 previous similar message [1191405.335813] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1191405.343126] Lustre: Skipped 1 previous similar message [1191698.201229] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1191698.208549] Lustre: Skipped 5 previous similar messages [1192450.615732] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [1192450.623049] Lustre: Skipped 11 previous similar messages [1193115.645040] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [1193115.652441] Lustre: Skipped 12 previous similar messages [1193742.190533] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [1193742.197855] Lustre: Skipped 9 previous similar messages [1194551.985273] Lustre: fir-OST0021: Connection restored to (at 10.8.6.33@o2ib6) [1194551.992587] Lustre: Skipped 4 previous similar messages [1200324.120235] md: md7: data-check done. [1201520.086485] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1201520.093796] Lustre: Skipped 4 previous similar messages [1209948.498740] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1209948.506065] Lustre: Skipped 1 previous similar message [1210526.811887] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1211451.740490] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1211451.747809] Lustre: Skipped 2 previous similar messages [1211454.460933] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1211454.468254] Lustre: Skipped 1 previous similar message [1211564.537023] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1212233.530558] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1214061.460809] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1214061.468129] Lustre: Skipped 1 previous similar message [1216547.161194] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1216547.168514] Lustre: Skipped 1 previous similar message [1216982.172588] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1217283.198293] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1217283.205610] Lustre: Skipped 1 previous similar message [1217582.206593] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1217582.213908] Lustre: Skipped 1 previous similar message [1217721.322695] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1217721.330016] Lustre: Skipped 1 previous similar message [1219375.652843] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1219375.660169] Lustre: Skipped 4 previous similar messages [1219376.865479] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1219683.248526] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1219983.284703] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1219983.292027] Lustre: Skipped 5 previous similar messages [1220283.294316] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1220283.301637] Lustre: Skipped 5 previous similar messages [1220582.302616] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1220582.309942] Lustre: Skipped 5 previous similar messages [1220882.309372] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1220882.316682] Lustre: Skipped 5 previous similar messages [1221182.317019] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1221182.324342] Lustre: Skipped 5 previous similar messages [1221482.322009] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1221482.329334] Lustre: Skipped 5 previous similar messages [1221782.339420] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1221782.346739] Lustre: Skipped 5 previous similar messages [1222383.410702] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1222383.418022] Lustre: Skipped 11 previous similar messages [1223282.368372] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1223282.375693] Lustre: Skipped 17 previous similar messages [1223882.385202] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1223882.392549] Lustre: Skipped 27 previous similar messages [1224482.402921] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1224482.410236] Lustre: Skipped 22 previous similar messages [1225382.363796] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1225382.371118] Lustre: Skipped 14 previous similar messages [1225983.380040] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1225983.387353] Lustre: Skipped 11 previous similar messages [1226883.403459] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1226883.410775] Lustre: Skipped 17 previous similar messages [1227782.428318] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1227782.435638] Lustre: Skipped 17 previous similar messages [1228382.447003] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1228382.454319] Lustre: Skipped 11 previous similar messages [1228982.463617] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1228982.470931] Lustre: Skipped 13 previous similar messages [1229582.538579] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1229582.545897] Lustre: Skipped 9 previous similar messages [1230482.560365] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1230482.567687] Lustre: Skipped 17 previous similar messages [1231082.578059] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1231082.585373] Lustre: Skipped 11 previous similar messages [1231983.540878] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1231983.548193] Lustre: Skipped 17 previous similar messages [1232882.565118] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1232882.572433] Lustre: Skipped 17 previous similar messages [1233482.583003] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1233482.590319] Lustre: Skipped 14 previous similar messages [1234082.657636] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1234082.664951] Lustre: Skipped 8 previous similar messages [1234682.676325] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1234682.683644] Lustre: Skipped 11 previous similar messages [1235282.693391] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1235282.700717] Lustre: Skipped 16 previous similar messages [1236182.697881] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1236182.705194] Lustre: Skipped 12 previous similar messages [1236783.730473] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1236783.737791] Lustre: Skipped 11 previous similar messages [1237683.751496] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1237683.758814] Lustre: Skipped 17 previous similar messages [1238283.769285] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1238283.776602] Lustre: Skipped 12 previous similar messages [1238883.788301] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1238883.795618] Lustre: Skipped 10 previous similar messages [1239782.809022] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1239782.816338] Lustre: Skipped 17 previous similar messages [1240382.826707] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1240382.834037] Lustre: Skipped 14 previous similar messages [1241282.787414] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1241282.794728] Lustre: Skipped 14 previous similar messages [1241882.865021] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1241882.872345] Lustre: Skipped 11 previous similar messages [1242482.883004] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1242482.890327] Lustre: Skipped 16 previous similar messages [1243382.844387] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1243382.851707] Lustre: Skipped 12 previous similar messages [1243982.920915] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1243982.928233] Lustre: Skipped 11 previous similar messages [1244883.879685] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1244883.887000] Lustre: Skipped 17 previous similar messages [1245782.970216] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1245782.977540] Lustre: Skipped 17 previous similar messages [1246382.998639] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1246383.005953] Lustre: Skipped 11 previous similar messages [1246983.998695] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1246984.006013] Lustre: Skipped 11 previous similar messages [1247883.024142] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1247883.031461] Lustre: Skipped 17 previous similar messages [1248484.039263] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1248484.046589] Lustre: Skipped 11 previous similar messages [1249084.057090] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1249084.064425] Lustre: Skipped 12 previous similar messages [1249984.077618] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1249984.084969] Lustre: Skipped 16 previous similar messages [1250883.104697] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1250883.112017] Lustre: Skipped 17 previous similar messages [1251783.127384] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1251783.134705] Lustre: Skipped 17 previous similar messages [1252683.117862] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1252683.125202] Lustre: Skipped 17 previous similar messages [1253583.141691] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1253583.149001] Lustre: Skipped 17 previous similar messages [1254184.152728] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1254184.160048] Lustre: Skipped 11 previous similar messages [1255083.176949] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1255083.184268] Lustre: Skipped 17 previous similar messages [1255683.193988] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1255683.201302] Lustre: Skipped 16 previous similar messages [1256283.240685] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1256283.248006] Lustre: Skipped 6 previous similar messages [1257183.203342] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1257183.210673] Lustre: Skipped 17 previous similar messages [1257783.221672] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1257783.228988] Lustre: Skipped 14 previous similar messages [1258383.292630] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1258383.299956] Lustre: Skipped 8 previous similar messages [1259283.258807] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1259283.266120] Lustre: Skipped 17 previous similar messages [1259883.276663] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1259883.283980] Lustre: Skipped 11 previous similar messages [1260483.293995] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1260483.301348] Lustre: Skipped 15 previous similar messages [1261383.331303] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1261383.338625] Lustre: Skipped 13 previous similar messages [1262283.355922] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1262283.363246] Lustre: Skipped 17 previous similar messages [1262883.373564] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1262883.380881] Lustre: Skipped 16 previous similar messages [1263484.386117] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1263484.393500] Lustre: Skipped 6 previous similar messages [1264383.409517] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1264383.416837] Lustre: Skipped 17 previous similar messages [1265283.418371] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1265283.425695] Lustre: Skipped 17 previous similar messages [1266183.500739] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1266183.508063] Lustre: Skipped 17 previous similar messages [1267083.508726] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1267083.516067] Lustre: Skipped 17 previous similar messages [1267983.506395] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1267983.513755] Lustre: Skipped 17 previous similar messages [1268584.516637] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1268584.523948] Lustre: Skipped 11 previous similar messages [1269484.538137] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1269484.545463] Lustre: Skipped 17 previous similar messages [1270383.605827] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1270383.613148] Lustre: Skipped 17 previous similar messages [1270983.623644] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1270983.630959] Lustre: Skipped 12 previous similar messages [1271884.582456] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1271884.589772] Lustre: Skipped 16 previous similar messages [1272484.644462] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1272484.651809] Lustre: Skipped 11 previous similar messages [1273384.683971] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1273384.691296] Lustre: Skipped 17 previous similar messages [1274283.706399] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1274283.713714] Lustre: Skipped 17 previous similar messages [1274883.725464] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1274883.732796] Lustre: Skipped 11 previous similar messages [1275484.739286] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1275484.746601] Lustre: Skipped 11 previous similar messages [1276384.763148] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1276384.770471] Lustre: Skipped 17 previous similar messages [1277283.786214] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1277283.793539] Lustre: Skipped 17 previous similar messages [1277883.803556] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1277883.810872] Lustre: Skipped 13 previous similar messages [1278783.764976] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1278783.772293] Lustre: Skipped 15 previous similar messages [1279683.788541] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1279683.795859] Lustre: Skipped 17 previous similar messages [1280583.875318] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1280583.882644] Lustre: Skipped 17 previous similar messages [1281184.886258] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1281184.893576] Lustre: Skipped 11 previous similar messages [1282084.908749] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1282084.916061] Lustre: Skipped 17 previous similar messages [1282983.932210] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1282983.939532] Lustre: Skipped 17 previous similar messages [1283600.103040] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1283600.113478] Lustre: Skipped 35 previous similar messages [1284483.967403] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1284483.974721] Lustre: Skipped 28 previous similar messages [1285141.871460] Lustre: fir-OST0019: Connection restored to d24c46ad-c948-5081-831c-1314be2aea23 (at 10.8.6.34@o2ib6) [1285141.881948] Lustre: Skipped 34 previous similar messages [1285984.004993] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1285984.012312] Lustre: Skipped 12 previous similar messages [1286584.023556] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1286584.030876] Lustre: Skipped 12 previous similar messages [1287185.033302] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1287185.040614] Lustre: Skipped 10 previous similar messages [1288084.056665] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1288084.063986] Lustre: Skipped 17 previous similar messages [1288684.075186] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1288684.082500] Lustre: Skipped 15 previous similar messages [1289119.101935] md: md9: data-check done. [1289285.090056] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1289285.097422] Lustre: Skipped 7 previous similar messages [1290184.110624] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1290184.117939] Lustre: Skipped 17 previous similar messages [1290784.128039] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1290784.135365] Lustre: Skipped 12 previous similar messages [1291684.088332] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1291684.095650] Lustre: Skipped 16 previous similar messages [1292284.164568] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1292284.171889] Lustre: Skipped 11 previous similar messages [1292885.181411] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1292885.188725] Lustre: Skipped 11 previous similar messages [1293784.203849] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1293784.211172] Lustre: Skipped 17 previous similar messages [1294385.219912] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1294385.227227] Lustre: Skipped 11 previous similar messages [1295284.240180] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1295284.247506] Lustre: Skipped 17 previous similar messages [1296184.202623] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1296184.209943] Lustre: Skipped 17 previous similar messages [1296784.220899] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1296784.228219] Lustre: Skipped 12 previous similar messages [1297385.236132] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1297385.243480] Lustre: Skipped 10 previous similar messages [1297985.312472] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1297985.319790] Lustre: Skipped 11 previous similar messages [1298884.333499] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1298884.340813] Lustre: Skipped 17 previous similar messages [1299484.351429] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1299484.358751] Lustre: Skipped 11 previous similar messages [1300084.368342] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1300084.375668] Lustre: Skipped 14 previous similar messages [1300984.390402] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1300984.397718] Lustre: Skipped 14 previous similar messages [1301584.409492] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1301584.416812] Lustre: Skipped 16 previous similar messages [1302185.420522] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1302185.427839] Lustre: Skipped 6 previous similar messages [1303084.445279] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1303084.452591] Lustre: Skipped 17 previous similar messages [1303685.459391] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1303685.466705] Lustre: Skipped 11 previous similar messages [1304584.481831] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1304584.489156] Lustre: Skipped 17 previous similar messages [1305484.504529] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1305484.511846] Lustre: Skipped 17 previous similar messages [1306384.526781] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1306384.534106] Lustre: Skipped 17 previous similar messages [1307284.515957] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1307284.523274] Lustre: Skipped 17 previous similar messages [1308184.509205] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1308184.516520] Lustre: Skipped 17 previous similar messages [1308784.527637] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1308784.534956] Lustre: Skipped 16 previous similar messages [1309385.600444] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1309385.607757] Lustre: Skipped 6 previous similar messages [1310284.563238] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1310284.570553] Lustre: Skipped 17 previous similar messages [1310884.639736] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1310884.647060] Lustre: Skipped 11 previous similar messages [1311784.601361] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1311784.608678] Lustre: Skipped 17 previous similar messages [1312384.645354] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1312384.652669] Lustre: Skipped 11 previous similar messages [1312984.694306] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1312984.701622] Lustre: Skipped 11 previous similar messages [1313585.707830] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1313585.715175] Lustre: Skipped 11 previous similar messages [1314485.730803] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1314485.738118] Lustre: Skipped 17 previous similar messages [1315385.754362] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1315385.761676] Lustre: Skipped 17 previous similar messages [1316285.776431] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1316285.783746] Lustre: Skipped 17 previous similar messages [1317184.797748] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1317184.805061] Lustre: Skipped 17 previous similar messages [1318085.758730] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1318085.766043] Lustre: Skipped 17 previous similar messages [1318685.776100] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1318685.783428] Lustre: Skipped 13 previous similar messages [1319584.795887] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1319584.803207] Lustre: Skipped 15 previous similar messages [1320184.813561] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1320184.820880] Lustre: Skipped 11 previous similar messages [1320785.887202] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1320785.894515] Lustre: Skipped 11 previous similar messages [1321684.910979] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1321684.918299] Lustre: Skipped 17 previous similar messages [1322584.873586] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1322584.880907] Lustre: Skipped 17 previous similar messages [1323185.888575] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1323185.895882] Lustre: Skipped 11 previous similar messages [1324084.912919] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1324084.920239] Lustre: Skipped 17 previous similar messages [1324684.931738] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1324684.939055] Lustre: Skipped 11 previous similar messages [1325285.005542] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1325285.012863] Lustre: Skipped 11 previous similar messages [1326185.966660] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1326185.973974] Lustre: Skipped 17 previous similar messages [1326785.985129] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1326785.992444] Lustre: Skipped 11 previous similar messages [1327685.005579] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1327685.012907] Lustre: Skipped 17 previous similar messages [1328285.023377] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1328285.030697] Lustre: Skipped 12 previous similar messages [1329186.106344] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1329186.113663] Lustre: Skipped 16 previous similar messages [1330085.129606] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1330085.136930] Lustre: Skipped 17 previous similar messages [1330985.089443] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1330985.096777] Lustre: Skipped 17 previous similar messages [1331885.112035] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1331885.119349] Lustre: Skipped 17 previous similar messages [1332485.130671] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1332485.137992] Lustre: Skipped 11 previous similar messages [1333085.206466] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1333085.213778] Lustre: Skipped 11 previous similar messages [1333686.223113] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1333686.230434] Lustre: Skipped 11 previous similar messages [1334585.247072] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1334585.254390] Lustre: Skipped 17 previous similar messages [1335486.211103] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1335486.218419] Lustre: Skipped 17 previous similar messages [1336385.233762] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1336385.241079] Lustre: Skipped 17 previous similar messages [1336985.251868] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1336985.259194] Lustre: Skipped 15 previous similar messages [1337585.269326] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1337585.276656] Lustre: Skipped 12 previous similar messages [1338186.342251] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1338186.349577] Lustre: Skipped 6 previous similar messages [1339085.367431] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1339085.374744] Lustre: Skipped 17 previous similar messages [1339686.382655] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1339686.389974] Lustre: Skipped 11 previous similar messages [1340585.408200] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1340585.415525] Lustre: Skipped 17 previous similar messages [1341185.426212] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1341185.433531] Lustre: Skipped 15 previous similar messages [1342085.442426] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1342085.449746] Lustre: Skipped 13 previous similar messages [1342685.461048] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1342685.468371] Lustre: Skipped 16 previous similar messages [1343585.423276] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1343585.430590] Lustre: Skipped 12 previous similar messages [1344185.440216] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1344185.447529] Lustre: Skipped 16 previous similar messages [1344786.453833] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1344786.461150] Lustre: Skipped 6 previous similar messages [1345686.479322] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1345686.486636] Lustre: Skipped 17 previous similar messages [1346585.503851] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1346585.511164] Lustre: Skipped 17 previous similar messages [1347185.521744] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1347185.529053] Lustre: Skipped 11 previous similar messages [1347786.537068] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1347786.544382] Lustre: Skipped 11 previous similar messages [1348386.554405] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1348386.561733] Lustre: Skipped 12 previous similar messages [1349285.576596] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1349285.583923] Lustre: Skipped 16 previous similar messages [1349885.594488] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1349885.601802] Lustre: Skipped 15 previous similar messages [1350485.635622] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1350485.642939] Lustre: Skipped 7 previous similar messages [1351085.684391] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1351085.691714] Lustre: Skipped 11 previous similar messages [1351985.648167] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1351985.655484] Lustre: Skipped 17 previous similar messages [1352886.730312] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1352886.737633] Lustre: Skipped 17 previous similar messages [1353486.748215] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1353486.755529] Lustre: Skipped 14 previous similar messages [1354385.769609] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1354385.776924] Lustre: Skipped 14 previous similar messages [1354985.786904] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1354985.794222] Lustre: Skipped 12 previous similar messages [1355585.849453] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1355585.856788] Lustre: Skipped 14 previous similar messages [1356186.815183] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1356186.822499] Lustre: Skipped 7 previous similar messages [1357085.839150] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1357085.846471] Lustre: Skipped 17 previous similar messages [1357685.857420] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1357685.864748] Lustre: Skipped 14 previous similar messages [1358585.818027] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1358585.825349] Lustre: Skipped 14 previous similar messages [1359186.833542] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1359186.840860] Lustre: Skipped 11 previous similar messages [1360085.857547] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1360085.864867] Lustre: Skipped 17 previous similar messages [1360686.873203] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1360686.880521] Lustre: Skipped 11 previous similar messages [1361286.948614] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1361286.955931] Lustre: Skipped 11 previous similar messages [1362185.973501] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1362185.980814] Lustre: Skipped 17 previous similar messages [1362785.991156] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1362785.998480] Lustre: Skipped 13 previous similar messages [1363687.011645] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1363687.018960] Lustre: Skipped 15 previous similar messages [1364586.037797] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1364586.045144] Lustre: Skipped 17 previous similar messages [1365186.055888] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1365186.063200] Lustre: Skipped 11 previous similar messages [1366087.019964] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1366087.027282] Lustre: Skipped 17 previous similar messages [1366987.101359] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1366987.108682] Lustre: Skipped 17 previous similar messages [1367587.119396] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1367587.126717] Lustre: Skipped 11 previous similar messages [1368487.143440] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1368487.150756] Lustre: Skipped 17 previous similar messages [1369386.167369] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1369386.174691] Lustre: Skipped 17 previous similar messages [1369986.185252] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1369986.192584] Lustre: Skipped 14 previous similar messages [1370886.144087] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1370886.151408] Lustre: Skipped 14 previous similar messages [1371786.167571] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1371786.174888] Lustre: Skipped 17 previous similar messages [1372386.185336] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1372386.192654] Lustre: Skipped 11 previous similar messages [1372986.202987] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1372986.210305] Lustre: Skipped 16 previous similar messages [1373587.211854] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1373587.219174] Lustre: Skipped 6 previous similar messages [1374486.235246] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1374486.242564] Lustre: Skipped 17 previous similar messages [1375086.253666] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1375086.260979] Lustre: Skipped 14 previous similar messages [1375687.265269] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1375687.272603] Lustre: Skipped 8 previous similar messages [1376587.292456] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1376587.299771] Lustre: Skipped 17 previous similar messages [1377486.373956] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1377486.381277] Lustre: Skipped 17 previous similar messages [1378086.391419] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1378086.398744] Lustre: Skipped 13 previous similar messages [1378986.412813] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1378986.420133] Lustre: Skipped 15 previous similar messages [1379587.426407] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1379587.433736] Lustre: Skipped 11 previous similar messages [1380486.450379] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1380486.457698] Lustre: Skipped 17 previous similar messages [1381386.411368] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1381386.418688] Lustre: Skipped 17 previous similar messages [1381986.490084] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1381986.497406] Lustre: Skipped 11 previous similar messages [1382587.505432] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1382587.512746] Lustre: Skipped 11 previous similar messages [1383486.463831] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1383486.471149] Lustre: Skipped 17 previous similar messages [1384086.539679] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1384086.546997] Lustre: Skipped 11 previous similar messages [1384687.555352] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1384687.562667] Lustre: Skipped 11 previous similar messages [1385287.572992] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1385287.580311] Lustre: Skipped 16 previous similar messages [1386187.594274] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1386187.601589] Lustre: Skipped 12 previous similar messages [1387086.614400] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1387086.621719] Lustre: Skipped 17 previous similar messages [1387986.635249] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1387986.642569] Lustre: Skipped 17 previous similar messages [1388586.653722] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1388586.661034] Lustre: Skipped 11 previous similar messages [1389486.675973] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1389486.683299] Lustre: Skipped 17 previous similar messages [1390087.689707] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1390087.697028] Lustre: Skipped 11 previous similar messages [1390986.712569] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1390986.719878] Lustre: Skipped 17 previous similar messages [1391887.734035] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1391887.741351] Lustre: Skipped 17 previous similar messages [1392786.754626] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1392786.761946] Lustre: Skipped 17 previous similar messages [1393386.771727] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1393386.779063] Lustre: Skipped 11 previous similar messages [1394286.792850] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1394286.800165] Lustre: Skipped 17 previous similar messages [1394886.809680] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1394886.816995] Lustre: Skipped 15 previous similar messages [1395787.830088] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1395787.837399] Lustre: Skipped 13 previous similar messages [1396686.854099] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1396686.861412] Lustre: Skipped 17 previous similar messages [1397287.867603] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1397287.874961] Lustre: Skipped 11 previous similar messages [1398186.892741] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1398186.900063] Lustre: Skipped 17 previous similar messages [1399086.882546] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1399086.889863] Lustre: Skipped 17 previous similar messages [1399686.928205] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1399686.935519] Lustre: Skipped 11 previous similar messages [1400286.944777] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1400286.952093] Lustre: Skipped 11 previous similar messages [1401186.965092] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1401186.972418] Lustre: Skipped 17 previous similar messages [1401786.987226] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1401786.994546] Lustre: Skipped 12 previous similar messages [1402686.942524] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1402686.949856] Lustre: Skipped 16 previous similar messages [1403287.018608] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1403287.025929] Lustre: Skipped 11 previous similar messages [1403887.036994] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1403887.044304] Lustre: Skipped 11 previous similar messages [1404488.050628] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1404488.057945] Lustre: Skipped 11 previous similar messages [1405388.074278] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1405388.081602] Lustre: Skipped 17 previous similar messages [1406288.100609] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1406288.107934] Lustre: Skipped 17 previous similar messages [1406732.065425] md: data-check of RAID array md1 [1406738.220502] md: data-check of RAID array md7 [1406744.219574] md: data-check of RAID array md9 [1406750.292327] md: data-check of RAID array md3 [1406756.364374] md: data-check of RAID array md11 [1406888.119724] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1406888.127102] Lustre: Skipped 13 previous similar messages [1407787.142127] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1407787.149450] Lustre: Skipped 15 previous similar messages [1408687.099861] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1408687.107178] Lustre: Skipped 17 previous similar messages [1409287.176991] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1409287.184386] Lustre: Skipped 11 previous similar messages [1409887.196059] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1409887.203385] Lustre: Skipped 14 previous similar messages [1410788.153369] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1410788.160678] Lustre: Skipped 14 previous similar messages [1411687.178329] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1411687.185715] Lustre: Skipped 17 previous similar messages [1412288.193014] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1412288.200341] Lustre: Skipped 11 previous similar messages [1413187.214833] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1413187.222243] Lustre: Skipped 17 previous similar messages [1413787.233589] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1413787.240912] Lustre: Skipped 13 previous similar messages [1414388.309942] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1414388.317269] Lustre: Skipped 9 previous similar messages [1415287.331869] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1415287.339213] Lustre: Skipped 17 previous similar messages [1415888.347217] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1415888.354536] Lustre: Skipped 11 previous similar messages [1416787.371316] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1416787.378660] Lustre: Skipped 17 previous similar messages [1417387.388994] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1417387.396374] Lustre: Skipped 12 previous similar messages [1417988.403411] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1417988.410739] Lustre: Skipped 10 previous similar messages [1418887.427291] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1418887.434635] Lustre: Skipped 17 previous similar messages [1419787.392324] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1419787.399649] Lustre: Skipped 17 previous similar messages [1420687.411717] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1420687.419057] Lustre: Skipped 17 previous similar messages [1421288.455634] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1421288.462956] Lustre: Skipped 11 previous similar messages [1422187.519224] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1422187.526577] Lustre: Skipped 17 previous similar messages [1423087.536676] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1423087.543989] Lustre: Skipped 17 previous similar messages [1423987.500201] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1423987.507524] Lustre: Skipped 17 previous similar messages [1424887.520071] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1424887.527399] Lustre: Skipped 17 previous similar messages [1425487.537178] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1425487.544517] Lustre: Skipped 12 previous similar messages [1426087.555051] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1426087.562392] Lustre: Skipped 13 previous similar messages [1426687.630127] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1426687.637470] Lustre: Skipped 8 previous similar messages [1427587.592952] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1427587.600289] Lustre: Skipped 17 previous similar messages [1428187.632465] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1428187.639792] Lustre: Skipped 11 previous similar messages [1429087.632724] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1429087.640056] Lustre: Skipped 17 previous similar messages [1429687.650628] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1429687.657975] Lustre: Skipped 15 previous similar messages [1430288.661364] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1430288.668681] Lustre: Skipped 7 previous similar messages [1431187.686951] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1431187.694274] Lustre: Skipped 17 previous similar messages [1431787.705315] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1431787.712637] Lustre: Skipped 14 previous similar messages [1432688.786588] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1432688.793910] Lustre: Skipped 14 previous similar messages [1433587.811058] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1433587.818389] Lustre: Skipped 17 previous similar messages [1434187.829171] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1434187.836508] Lustre: Skipped 13 previous similar messages [1434787.847695] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1434787.855206] Lustre: Skipped 13 previous similar messages [1435688.865331] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1435688.872643] Lustre: Skipped 13 previous similar messages [1436587.889763] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1436587.897082] Lustre: Skipped 17 previous similar messages [1437187.907878] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1437187.915242] Lustre: Skipped 13 previous similar messages [1438087.894601] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1438087.901926] Lustre: Skipped 15 previous similar messages [1438688.943537] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1438688.950869] Lustre: Skipped 11 previous similar messages [1439587.968099] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1439587.975428] Lustre: Skipped 17 previous similar messages [1440188.982739] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1440188.990078] Lustre: Skipped 11 previous similar messages [1441089.003160] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1441089.010489] Lustre: Skipped 17 previous similar messages [1441689.021477] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1441689.028802] Lustre: Skipped 15 previous similar messages [1442587.982963] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1442587.990300] Lustre: Skipped 13 previous similar messages [1443188.062290] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1443188.069614] Lustre: Skipped 11 previous similar messages [1444088.085086] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1444088.092397] Lustre: Skipped 17 previous similar messages [1444688.103502] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1444688.110840] Lustre: Skipped 15 previous similar messages [1445588.102529] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1445588.109858] Lustre: Skipped 13 previous similar messages [1446189.083970] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1446189.091284] Lustre: Skipped 11 previous similar messages [1447088.099679] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1447088.106995] Lustre: Skipped 17 previous similar messages [1447688.118047] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1447688.125412] Lustre: Skipped 11 previous similar messages [1448589.201647] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1448589.208983] Lustre: Skipped 17 previous similar messages [1449488.229205] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1449488.236522] Lustre: Skipped 17 previous similar messages [1450389.248773] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1450389.256101] Lustre: Skipped 17 previous similar messages [1451288.210704] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1451288.218054] Lustre: Skipped 17 previous similar messages [1451888.290111] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1451888.297434] Lustre: Skipped 11 previous similar messages [1452788.250735] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1452788.258066] Lustre: Skipped 17 previous similar messages [1453688.273103] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1453688.280443] Lustre: Skipped 17 previous similar messages [1454289.287219] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1454289.294539] Lustre: Skipped 11 previous similar messages [1455188.371973] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1455188.379304] Lustre: Skipped 17 previous similar messages [1456089.397926] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1456089.405245] Lustre: Skipped 17 previous similar messages [1456988.420421] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1456988.427759] Lustre: Skipped 17 previous similar messages [1457888.409229] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1457888.416590] Lustre: Skipped 17 previous similar messages [1458488.455874] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1458488.463201] Lustre: Skipped 11 previous similar messages [1459088.486883] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1459088.494233] Lustre: Skipped 15 previous similar messages [1459989.433975] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1459989.441408] Lustre: Skipped 13 previous similar messages [1460589.451833] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1460589.459174] Lustre: Skipped 11 previous similar messages [1461189.527608] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1461189.534927] Lustre: Skipped 11 previous similar messages [1462088.552147] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1462088.559494] Lustre: Skipped 17 previous similar messages [1462688.569701] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1462688.577041] Lustre: Skipped 11 previous similar messages [1463288.587564] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1463288.594923] Lustre: Skipped 16 previous similar messages [1464189.551462] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1464189.558809] Lustre: Skipped 12 previous similar messages [1465088.571728] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1465088.579054] Lustre: Skipped 17 previous similar messages [1465688.589520] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1465688.596949] Lustre: Skipped 12 previous similar messages [1466289.603807] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1466289.611133] Lustre: Skipped 10 previous similar messages [1467188.659557] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1467188.666891] Lustre: Skipped 17 previous similar messages [1468088.712025] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1468088.719341] Lustre: Skipped 17 previous similar messages [1468988.674075] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1468988.681468] Lustre: Skipped 17 previous similar messages [1469588.691911] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1469588.699233] Lustre: Skipped 12 previous similar messages [1470189.769374] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1470189.776761] Lustre: Skipped 10 previous similar messages [1471088.791742] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1471088.799067] Lustre: Skipped 17 previous similar messages [1471688.809327] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1471688.816647] Lustre: Skipped 15 previous similar messages [1472289.820222] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1472289.827545] Lustre: Skipped 7 previous similar messages [1473189.843650] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1473189.850996] Lustre: Skipped 17 previous similar messages [1474088.870419] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1474088.877745] Lustre: Skipped 17 previous similar messages [1474689.884035] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1474689.891353] Lustre: Skipped 11 previous similar messages [1475589.844773] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1475589.852132] Lustre: Skipped 17 previous similar messages [1476189.921379] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1476189.928713] Lustre: Skipped 11 previous similar messages [1477089.947838] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1477089.955193] Lustre: Skipped 17 previous similar messages [1477988.971365] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1477988.978809] Lustre: Skipped 17 previous similar messages [1478588.989648] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1478588.997038] Lustre: Skipped 11 previous similar messages [1479490.009386] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1479490.016746] Lustre: Skipped 17 previous similar messages [1480389.036855] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1480389.044198] Lustre: Skipped 17 previous similar messages [1481289.995816] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1481290.003139] Lustre: Skipped 17 previous similar messages [1482189.018887] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1482189.026214] Lustre: Skipped 17 previous similar messages [1482790.036738] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1482790.044065] Lustre: Skipped 11 previous similar messages [1483690.058086] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1483690.065406] Lustre: Skipped 17 previous similar messages [1484589.082500] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1484589.089812] Lustre: Skipped 17 previous similar messages [1485489.139358] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1485489.146720] Lustre: Skipped 17 previous similar messages [1486389.160405] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1486389.167798] Lustre: Skipped 17 previous similar messages [1487289.153419] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1487289.160731] Lustre: Skipped 17 previous similar messages [1488189.174992] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1488189.182346] Lustre: Skipped 17 previous similar messages [1488789.193362] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1488789.200709] Lustre: Skipped 16 previous similar messages [1489389.271345] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1489389.278690] Lustre: Skipped 6 previous similar messages [1490289.232797] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1490289.240159] Lustre: Skipped 17 previous similar messages [1491189.253837] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1491189.261155] Lustre: Skipped 17 previous similar messages [1491789.270872] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1491789.278196] Lustre: Skipped 16 previous similar messages [1492689.290585] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1492689.297908] Lustre: Skipped 12 previous similar messages [1493589.314874] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1493589.322206] Lustre: Skipped 17 previous similar messages [1494189.333980] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1494189.341339] Lustre: Skipped 16 previous similar messages [1495089.351521] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1495089.358859] Lustre: Skipped 12 previous similar messages [1495689.368511] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1495689.375872] Lustre: Skipped 11 previous similar messages [1496289.386888] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1496289.394252] Lustre: Skipped 11 previous similar messages [1497190.471188] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1497190.478508] Lustre: Skipped 17 previous similar messages [1497790.488730] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1497790.496050] Lustre: Skipped 11 previous similar messages [1498690.509532] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1498690.516856] Lustre: Skipped 17 previous similar messages [1499589.533173] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1499589.540502] Lustre: Skipped 17 previous similar messages [1500190.553622] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1500190.560938] Lustre: Skipped 11 previous similar messages [1501090.512464] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1501090.519778] Lustre: Skipped 17 previous similar messages [1501989.535729] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1501989.543054] Lustre: Skipped 17 previous similar messages [1502702.102746] md: md1: data-check done. [1502889.559481] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1502889.566902] Lustre: Skipped 17 previous similar messages [1503489.576715] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1503489.584033] Lustre: Skipped 16 previous similar messages [1504389.658450] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1504389.665786] Lustre: Skipped 12 previous similar messages [1504439.117343] md: md3: data-check done. [1504989.675845] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1504989.683157] Lustre: Skipped 11 previous similar messages [1505889.699182] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1505889.706533] Lustre: Skipped 17 previous similar messages [1506114.304125] md: md11: data-check done. [1506490.713566] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1506490.720886] Lustre: Skipped 11 previous similar messages [1507090.730563] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1507090.737875] Lustre: Skipped 15 previous similar messages [1507989.756842] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1507989.764160] Lustre: Skipped 13 previous similar messages [1508590.771441] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1508590.778761] Lustre: Skipped 11 previous similar messages [1509489.797220] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1509489.804541] Lustre: Skipped 17 previous similar messages [1510090.810088] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1510090.817413] Lustre: Skipped 11 previous similar messages [1510989.834135] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1510989.841464] Lustre: Skipped 17 previous similar messages [1511589.853209] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1511589.860525] Lustre: Skipped 14 previous similar messages [1512189.871257] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1512189.878576] Lustre: Skipped 9 previous similar messages [1513089.892553] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1513089.899947] Lustre: Skipped 16 previous similar messages [1513690.909754] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1513690.917066] Lustre: Skipped 11 previous similar messages [1514589.934196] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1514589.941507] Lustre: Skipped 17 previous similar messages [1515189.953124] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1515189.960445] Lustre: Skipped 16 previous similar messages [1516089.957112] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1516089.964428] Lustre: Skipped 12 previous similar messages [1516689.974188] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1516689.981517] Lustre: Skipped 13 previous similar messages [1517290.002961] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1517290.010272] Lustre: Skipped 10 previous similar messages [1518189.967201] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1518189.974519] Lustre: Skipped 16 previous similar messages [1518789.984410] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1518789.991725] Lustre: Skipped 22 previous similar messages [1519390.002192] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1519390.009508] Lustre: Skipped 11 previous similar messages [1520291.084904] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1520291.092218] Lustre: Skipped 12 previous similar messages [1521190.109570] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1521190.116878] Lustre: Skipped 17 previous similar messages [1521790.126618] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1521790.133962] Lustre: Skipped 16 previous similar messages [1522691.147343] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1522691.154684] Lustre: Skipped 12 previous similar messages [1523591.172972] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1523591.180283] Lustre: Skipped 17 previous similar messages [1524490.164869] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1524490.172214] Lustre: Skipped 17 previous similar messages [1525090.212138] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1525090.219476] Lustre: Skipped 17 previous similar messages [1525775.157179] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1525775.164495] Lustre: Skipped 23 previous similar messages [1526590.253706] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1526590.261124] Lustre: Skipped 17 previous similar messages [1527190.270661] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1527190.277992] Lustre: Skipped 19 previous similar messages [1528091.229265] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1528091.236605] Lustre: Skipped 15 previous similar messages [1528991.315819] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1528991.323173] Lustre: Skipped 17 previous similar messages [1529591.333819] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1529591.341141] Lustre: Skipped 16 previous similar messages [1530490.357571] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1530490.364969] Lustre: Skipped 12 previous similar messages [1531390.318299] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1531390.325613] Lustre: Skipped 17 previous similar messages [1531991.335142] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1531991.342481] Lustre: Skipped 11 previous similar messages [1532591.414517] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1532591.421829] Lustre: Skipped 11 previous similar messages [1533491.436632] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1533491.443941] Lustre: Skipped 17 previous similar messages [1534390.461890] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1534390.469225] Lustre: Skipped 17 previous similar messages [1534990.479770] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1534990.487119] Lustre: Skipped 13 previous similar messages [1535590.496894] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1535590.504217] Lustre: Skipped 14 previous similar messages [1536490.456400] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1536490.463743] Lustre: Skipped 12 previous similar messages [1537090.474975] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1537090.482288] Lustre: Skipped 11 previous similar messages [1537690.553320] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1537690.560655] Lustre: Skipped 11 previous similar messages [1538290.570751] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1538290.578066] Lustre: Skipped 15 previous similar messages [1538891.584638] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1538891.591958] Lustre: Skipped 7 previous similar messages [1539491.601643] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1539491.608990] Lustre: Skipped 18 previous similar messages [1540091.618929] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1540091.626252] Lustre: Skipped 16 previous similar messages [1540990.641748] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1540990.649112] Lustre: Skipped 31 previous similar messages [1541591.656093] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1541591.663410] Lustre: Skipped 33 previous similar messages [1542221.239107] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1542221.246455] Lustre: Skipped 20 previous similar messages [1542822.672800] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1542822.680155] Lustre: Skipped 54 previous similar messages [1543690.694302] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1543690.701665] Lustre: Skipped 24 previous similar messages [1544290.712391] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1544290.719702] Lustre: Skipped 27 previous similar messages [1544940.391257] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1544940.398574] Lustre: Skipped 29 previous similar messages [1545586.589388] Lustre: fir-OST0023: Connection restored to 9b3a0481-8c2f-a606-18ad-7a06b856368e (at 10.8.3.11@o2ib6) [1545586.599830] Lustre: Skipped 66 previous similar messages [1546223.065591] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1546223.072942] Lustre: Skipped 41 previous similar messages [1546846.769219] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1546846.776536] Lustre: Skipped 28 previous similar messages [1547450.410298] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1547450.417610] Lustre: Skipped 36 previous similar messages [1548053.265607] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1548053.272923] Lustre: Skipped 37 previous similar messages [1548666.433542] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1548666.440895] Lustre: Skipped 42 previous similar messages [1549390.800131] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1549390.807456] Lustre: Skipped 47 previous similar messages [1549990.860555] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1549990.867879] Lustre: Skipped 11 previous similar messages [1550590.879310] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1550590.886675] Lustre: Skipped 11 previous similar messages [1551190.896303] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1551190.903622] Lustre: Skipped 19 previous similar messages [1551790.914065] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1551790.921455] Lustre: Skipped 24 previous similar messages [1552586.110643] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1552586.121094] Lustre: Skipped 17 previous similar messages [1553290.907222] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1553290.914539] Lustre: Skipped 12 previous similar messages [1553891.919925] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1553891.927245] Lustre: Skipped 21 previous similar messages [1554519.311912] Lustre: fir-OST0019: Connection restored to (at 10.8.9.6@o2ib6) [1554519.319154] Lustre: Skipped 46 previous similar messages [1555390.992479] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1555390.999797] Lustre: Skipped 17 previous similar messages [1555991.009852] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1555991.017196] Lustre: Skipped 15 previous similar messages [1556592.023497] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1556592.030817] Lustre: Skipped 13 previous similar messages [1557492.047820] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1557492.055133] Lustre: Skipped 17 previous similar messages [1558092.065241] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1558092.072601] Lustre: Skipped 17 previous similar messages [1558992.088784] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1558992.096144] Lustre: Skipped 17 previous similar messages [1559655.302225] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1559655.312678] Lustre: Skipped 27 previous similar messages [1559690.547248] md: md9: data-check done. [1559884.844639] md: md7: data-check done. [1560491.155904] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1560491.163222] Lustre: Skipped 12 previous similar messages [1561365.819787] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1561365.827108] Lustre: Skipped 20 previous similar messages [1561991.196235] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1561991.203566] Lustre: Skipped 15 previous similar messages [1562591.213688] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1562591.221013] Lustre: Skipped 16 previous similar messages [1563304.843709] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1563304.851032] Lustre: Skipped 28 previous similar messages [1563917.790324] Lustre: fir-OST0019: Connection restored to (at 10.8.6.33@o2ib6) [1563917.797644] Lustre: Skipped 24 previous similar messages [1564691.266783] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1564691.274101] Lustre: Skipped 23 previous similar messages [1565291.285051] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1565291.292370] Lustre: Skipped 12 previous similar messages [1565891.302957] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1565891.310268] Lustre: Skipped 25 previous similar messages [1566791.264867] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1566791.272185] Lustre: Skipped 23 previous similar messages [1567691.286236] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1567691.293543] Lustre: Skipped 23 previous similar messages [1568291.304145] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1568291.311463] Lustre: Skipped 16 previous similar messages [1569043.926172] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1569043.936629] Lustre: Skipped 30 previous similar messages [1569792.339574] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1569792.346889] Lustre: Skipped 18 previous similar messages [1570392.356956] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1570392.364272] Lustre: Skipped 28 previous similar messages [1571055.253886] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1571055.261196] Lustre: Skipped 34 previous similar messages [1571759.662516] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1571759.669833] Lustre: Skipped 12 previous similar messages [1572410.320148] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1572410.327474] Lustre: Skipped 18 previous similar messages [1573091.416788] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1573091.424103] Lustre: Skipped 22 previous similar messages [1573991.453026] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1573991.460347] Lustre: Skipped 18 previous similar messages [1574592.466857] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1574592.474169] Lustre: Skipped 18 previous similar messages [1575491.541166] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1575491.548498] Lustre: Skipped 23 previous similar messages [1576091.569645] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1576091.576959] Lustre: Skipped 17 previous similar messages [1576691.587686] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1576691.595000] Lustre: Skipped 12 previous similar messages [1577591.567193] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1577591.574509] Lustre: Skipped 16 previous similar messages [1578191.592982] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1578191.600317] Lustre: Skipped 17 previous similar messages [1579091.604752] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1579091.612063] Lustre: Skipped 19 previous similar messages [1579991.659687] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1579991.666999] Lustre: Skipped 23 previous similar messages [1580891.696462] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1580891.703778] Lustre: Skipped 17 previous similar messages [1581791.660453] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1581791.667763] Lustre: Skipped 23 previous similar messages [1582391.680314] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1582391.687634] Lustre: Skipped 11 previous similar messages [1583292.760880] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1583292.768201] Lustre: Skipped 23 previous similar messages [1584192.784481] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1584192.791793] Lustre: Skipped 17 previous similar messages [1585092.810615] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1585092.817930] Lustre: Skipped 23 previous similar messages [1585692.827620] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1585692.834935] Lustre: Skipped 14 previous similar messages [1586592.851106] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1586592.858424] Lustre: Skipped 20 previous similar messages [1587491.875117] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1587491.882439] Lustre: Skipped 17 previous similar messages [1588092.889844] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1588092.897158] Lustre: Skipped 17 previous similar messages [1588991.915165] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1588991.922482] Lustre: Skipped 17 previous similar messages [1589592.930144] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1589592.937456] Lustre: Skipped 11 previous similar messages [1590492.952222] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1590492.959540] Lustre: Skipped 23 previous similar messages [1591325.424891] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1591325.435333] Lustre: Skipped 22 previous similar messages [1591991.996433] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1591992.003751] Lustre: Skipped 12 previous similar messages [1592593.010361] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1592593.017675] Lustre: Skipped 11 previous similar messages [1593193.029115] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1593193.036432] Lustre: Skipped 17 previous similar messages [1594092.050047] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1594092.057364] Lustre: Skipped 16 previous similar messages [1594693.067044] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1594693.074359] Lustre: Skipped 11 previous similar messages [1595293.084754] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1595293.092071] Lustre: Skipped 22 previous similar messages [1596192.107018] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1596192.114331] Lustre: Skipped 12 previous similar messages [1596793.122543] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1596793.129852] Lustre: Skipped 17 previous similar messages [1597692.147023] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1597692.154336] Lustre: Skipped 17 previous similar messages [1598292.164514] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1598292.171834] Lustre: Skipped 16 previous similar messages [1599192.185976] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1599192.193295] Lustre: Skipped 18 previous similar messages [1599793.203232] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1599793.210554] Lustre: Skipped 17 previous similar messages [1600692.228073] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1600692.235388] Lustre: Skipped 17 previous similar messages [1601592.192283] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1601592.199601] Lustre: Skipped 23 previous similar messages [1602492.276186] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1602492.283518] Lustre: Skipped 23 previous similar messages [1603092.293385] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1603092.300703] Lustre: Skipped 14 previous similar messages [1603693.307095] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1603693.314409] Lustre: Skipped 14 previous similar messages [1604593.331520] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1604593.338835] Lustre: Skipped 17 previous similar messages [1605193.350014] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1605193.357330] Lustre: Skipped 28 previous similar messages [1606092.373020] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1606092.380333] Lustre: Skipped 12 previous similar messages [1606992.335954] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1606992.343270] Lustre: Skipped 23 previous similar messages [1607592.410975] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1607592.418306] Lustre: Skipped 11 previous similar messages [1608192.429351] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1608192.436664] Lustre: Skipped 18 previous similar messages [1608792.446423] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1608792.453754] Lustre: Skipped 13 previous similar messages [1609515.157924] Lustre: fir-OST0019: Connection restored to 85c45b71-10aa-be9f-071b-9a9c50548db2 (at 10.8.15.7@o2ib6) [1609515.168379] Lustre: Skipped 19 previous similar messages [1610292.422534] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1610292.429852] Lustre: Skipped 12 previous similar messages [1610893.440009] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1610893.447323] Lustre: Skipped 17 previous similar messages [1611792.461883] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1611792.469201] Lustre: Skipped 41 previous similar messages [1612393.477885] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1612393.485201] Lustre: Skipped 17 previous similar messages [1612993.495412] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1612993.502726] Lustre: Skipped 41 previous similar messages [1613893.577075] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1613893.584384] Lustre: Skipped 12 previous similar messages [1614707.292996] Lustre: fir-OST0019: Connection restored to 28764730-adc8-1b1a-0aa7-9bb3363d4889 (at 10.9.101.59@o2ib4) [1614707.303615] Lustre: Skipped 46 previous similar messages [1615388.228117] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1615388.235431] Lustre: Skipped 19 previous similar messages [1615992.634506] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1615992.641819] Lustre: Skipped 12 previous similar messages [1616593.652097] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1616593.659416] Lustre: Skipped 11 previous similar messages [1617378.143121] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1617378.150438] Lustre: Skipped 59 previous similar messages [1617921.443164] Lustre: fir-OST001f: haven't heard from client f844fdc2-4b3f-a5e3-d40e-580f6f5c3bcb (at 10.9.101.59@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92887165c800, cur 1549395585 expire 1549395435 last 1549395358 [1617921.465128] Lustre: Skipped 5 previous similar messages [1617924.993177] Lustre: fir-OST0019: haven't heard from client f844fdc2-4b3f-a5e3-d40e-580f6f5c3bcb (at 10.9.101.59@o2ib4) in 231 seconds. I think it's dead, and I am evicting it. exp ffff92887165c000, cur 1549395589 expire 1549395439 last 1549395358 [1617983.042967] Lustre: fir-OST0019: Connection restored to 2708a86e-3625-5509-9d0d-aaed4fc61a05 (at 10.8.1.31@o2ib6) [1617983.053406] Lustre: Skipped 35 previous similar messages [1618693.709524] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1618693.716842] Lustre: Skipped 36 previous similar messages [1619304.858685] Lustre: fir-OST0019: Connection restored to 3c005caf-e304-dd1a-234a-65ceb646a930 (at 10.8.1.2@o2ib6) [1619304.869048] Lustre: Skipped 22 previous similar messages [1619929.697144] Lustre: fir-OST001b: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1619929.707591] Lustre: Skipped 23 previous similar messages [1620837.938759] Lustre: fir-OST0023: haven't heard from client 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff928458ac5000, cur 1549398502 expire 1549398352 last 1549398275 [1621463.882069] Lustre: fir-OST0019: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1621463.892510] Lustre: Skipped 101 previous similar messages [1621654.034132] Lustre: fir-OST0019: Connection restored to 775ad82c-450c-c16e-bb21-700cda6d4afd (at 10.9.103.39@o2ib4) [1621654.044740] Lustre: Skipped 17 previous similar messages [1623027.779704] Lustre: fir-OST0019: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1623027.790142] Lustre: Skipped 17 previous similar messages [1623050.106794] Lustre: fir-OST0019: Connection restored to 8ae4cc68-204d-d2bb-4908-94f49692744b (at 10.8.20.15@o2ib6) [1623050.117335] Lustre: Skipped 11 previous similar messages [1623881.861096] Lustre: fir-OST0019: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [1623881.871449] Lustre: Skipped 5 previous similar messages [1624103.889783] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [1624103.900123] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Skipped 2 previous similar messages [1624103.910465] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.201@o2ib7 (46): c: 4, oc: 0, rc: 8 [1624103.922708] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Skipped 2 previous similar messages [1624271.710114] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1624275.354229] Lustre: fir-OST001b: haven't heard from client e288d2fa-1494-4b6c-352f-7c16658a712b (at 10.8.1.1@o2ib6) in 229 seconds. I think it's dead, and I am evicting it. exp ffff9280d1089400, cur 1549401939 expire 1549401789 last 1549401710 [1624275.375928] Lustre: Skipped 5 previous similar messages [1624276.735573] Lustre: fir-OST001d: haven't heard from client e288d2fa-1494-4b6c-352f-7c16658a712b (at 10.8.1.1@o2ib6) in 230 seconds. I think it's dead, and I am evicting it. exp ffff926a46ec1800, cur 1549401940 expire 1549401790 last 1549401710 [1624276.757282] Lustre: Skipped 6 previous similar messages [1624399.896624] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 296 seconds [1624399.907144] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 9 previous similar messages [1624402.896711] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.201@o2ib7: 0 seconds [1624402.907056] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [1624701.903641] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 598 seconds [1624701.914159] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1625001.910525] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 898 seconds [1625001.921043] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [1625303.917506] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 1200 seconds [1625303.928113] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [1626205.938331] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.210@o2ib7: 2102 seconds [1626210.938450] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.202@o2ib7: 0 seconds [1626210.948793] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [1626555.212541] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1626555.222900] Lustre: Skipped 5 previous similar messages [1626560.439773] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1626560.450124] Lustre: Skipped 2 previous similar messages [1626564.976920] Lustre: fir-OST001b: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1626578.296363] Lustre: fir-OST0023: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1627888.123618] Lustre: fir-OST0019: Connection restored to 6da5269e-e6e7-e930-ea8b-e990b1fd18b0 (at 10.9.101.72@o2ib4) [1627888.134229] Lustre: Skipped 5 previous similar messages [1627889.392708] Lustre: fir-OST0019: Connection restored to fd4739f5-61ac-b1c8-a4a0-9ad8b819daba (at 10.9.101.38@o2ib4) [1627889.403318] Lustre: Skipped 29 previous similar messages [1627891.803133] Lustre: fir-OST0019: Connection restored to 7ac937e4-43b1-06ba-e012-59210bc9ec60 (at 10.9.101.18@o2ib4) [1627891.813757] Lustre: Skipped 101 previous similar messages [1627948.030083] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1627948.037399] Lustre: Skipped 36 previous similar messages [1627987.958915] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1627987.966230] Lustre: Skipped 1 previous similar message [1628074.049832] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1628074.057164] Lustre: Skipped 3 previous similar messages [1628292.964548] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1628592.979201] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1628592.986518] Lustre: Skipped 5 previous similar messages [1628892.988870] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1628892.996183] Lustre: Skipped 5 previous similar messages [1629192.996479] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1629193.003807] Lustre: Skipped 5 previous similar messages [1629792.970821] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1629792.978141] Lustre: Skipped 74 previous similar messages [1630392.994566] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1630393.001885] Lustre: Skipped 400 previous similar messages [1630999.169692] Lustre: fir-OST0019: Connection restored to f836dc38-744c-c596-c0a2-f4034f572070 (at 10.9.102.15@o2ib4) [1630999.180302] Lustre: Skipped 82 previous similar messages [1631615.419677] Lustre: fir-OST0019: Connection restored to 75d3df1c-c05f-80f2-4e0d-3991a06b4cf2 (at 10.8.2.9@o2ib6) [1631615.430023] Lustre: Skipped 610 previous similar messages [1632218.393050] Lustre: fir-OST0019: Connection restored to 3ce6d475-6724-c117-8c44-da8378e50030 (at 10.9.101.69@o2ib4) [1632218.403659] Lustre: Skipped 352 previous similar messages [1632869.822220] Lustre: fir-OST001d: Connection restored to fd4739f5-61ac-b1c8-a4a0-9ad8b819daba (at 10.9.101.38@o2ib4) [1632869.832833] Lustre: Skipped 54 previous similar messages [1633512.858906] Lustre: fir-OST001d: Connection restored to 4944a2ce-d92c-784e-9536-c95b01530191 (at 10.9.101.48@o2ib4) [1633512.869523] Lustre: Skipped 29 previous similar messages [1634167.950072] Lustre: fir-OST001f: Connection restored to 0efd883e-065d-0727-44f7-75a6230140a1 (at 10.9.0.2@o2ib4) [1634167.960455] Lustre: Skipped 53 previous similar messages [1634793.244559] Lustre: fir-OST001f: Connection restored to 657c4193-574d-7ac0-e927-0684806e81d2 (at 10.9.101.42@o2ib4) [1634793.255171] Lustre: Skipped 46 previous similar messages [1635450.741762] Lustre: fir-OST001b: Connection restored to 45f81bad-ba79-a5b8-97f4-c718cac35552 (at 10.8.0.82@o2ib6) [1635450.752247] Lustre: Skipped 53 previous similar messages [1636055.150159] Lustre: fir-OST001b: Connection restored to e033b2fb-58ee-ad20-dbe1-c069873ac977 (at 10.9.101.47@o2ib4) [1636055.160837] Lustre: Skipped 60 previous similar messages [1637144.078723] Lustre: fir-OST001b: Connection restored to 61d0fcbf-b5c3-a78f-9cdf-55e61d025648 (at 10.9.0.81@o2ib4) [1637144.089213] Lustre: Skipped 23 previous similar messages [1638520.335908] Lustre: fir-OST0023: haven't heard from client 673a177f-1230-a616-2a0b-8995b4882735 (at 10.9.0.1@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92647953a800, cur 1549416184 expire 1549416034 last 1549415957 [1638520.357643] Lustre: Skipped 35 previous similar messages [1638568.806354] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1638568.816704] Lustre: Skipped 7 previous similar messages [1638987.509731] Lustre: fir-OST001d: Connection restored to 948ddf77-30e8-6f7f-29c3-d9d6fe8d8435 (at 10.9.101.4@o2ib4) [1638987.520258] Lustre: Skipped 4 previous similar messages [1639175.908705] Lustre: fir-OST0019: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1639175.919237] Lustre: Skipped 24 previous similar messages [1639498.604668] Lustre: fir-OST0023: Connection restored to 948ddf77-30e8-6f7f-29c3-d9d6fe8d8435 (at 10.9.101.4@o2ib4) [1639498.615191] Lustre: Skipped 20 previous similar messages [1639647.149690] Lustre: 46949:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417303/real 1549417303] req@ffff927fee582d00 x1623662782418864/t0(0) o104->fir-OST0023@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417310 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1639647.177003] Lustre: 46949:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 32 previous similar messages [1639652.253807] Lustre: 37571:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417308/real 1549417308] req@ffff92829beb9e00 x1623662782422464/t0(0) o104->fir-OST001f@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417315 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1639652.281057] Lustre: 37571:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [1639654.149855] Lustre: 69439:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417310/real 1549417310] req@ffff92780c66d100 x1623662782418848/t0(0) o104->fir-OST0023@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417317 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639654.149857] Lustre: 118101:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417310/real 1549417310] req@ffff92763cf5e900 x1623662782419008/t0(0) o104->fir-OST0023@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417317 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639654.149862] Lustre: 118101:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 10 previous similar messages [1639654.214542] Lustre: 69439:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 32 previous similar messages [1639659.268979] Lustre: 92985:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417315/real 1549417315] req@ffff928431bee000 x1623662782422704/t0(0) o104->fir-OST001f@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417322 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639659.296229] Lustre: 92985:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [1639666.270133] Lustre: 64309:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417322/real 1549417322] req@ffff9280b0274200 x1623662782423072/t0(0) o104->fir-OST001f@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417329 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639666.297386] Lustre: 64309:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 46 previous similar messages [1639675.150336] Lustre: 118101:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417331/real 1549417331] req@ffff92763cf5e900 x1623662782419008/t0(0) o104->fir-OST0023@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417338 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639675.177702] Lustre: 118101:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 86 previous similar messages [1639694.272787] Lustre: 115818:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417350/real 1549417350] req@ffff92780c66b300 x1623662782423536/t0(0) o104->fir-OST0019@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417357 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639694.300148] Lustre: 115818:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 94 previous similar messages [1639729.292590] Lustre: 37571:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417385/real 1549417385] req@ffff92829beb9e00 x1623662782422464/t0(0) o104->fir-OST001f@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417392 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639729.319839] Lustre: 37571:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 235 previous similar messages [1639745.152976] LustreError: 37562:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) ### client (nid 10.9.0.2@o2ib4) failed to reply to blocking AST (req@ffff9283ef2c3900 x1623662782419120 status 0 rc -110), evict it ns: filter-fir-OST0023_UUID lock: ffff92668d44dc40/0x4162c98b034e1a7c lrc: 4/0,0 mode: PR/PR res: [0x30de2:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c29d2b8 expref: 30 pid: 13296 timeout: 1639799 lvb_type: 1 [1639745.154974] LustreError: 138-a: fir-OST0023: A client on nid 10.9.0.2@o2ib4 was evicted due to a lock blocking callback time out: rc -110 [1639745.154994] LustreError: 63402:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 105s: evicting client at 10.9.0.2@o2ib4 ns: filter-fir-OST0023_UUID lock: ffff92700cf12880/0x4162c98b034e21c2 lrc: 3/0,0 mode: PR/PR res: [0x30e90:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c2a041a expref: 31 pid: 13296 timeout: 0 lvb_type: 1 [1639745.250701] LustreError: 37562:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) Skipped 6 previous similar messages [1639794.155089] Lustre: 69446:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549417450/real 1549417450] req@ffff9280b0272a00 x1623662782419488/t0(0) o104->fir-OST001d@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549417457 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1639794.156105] LustreError: 69449:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) ### client (nid 10.9.0.2@o2ib4) failed to reply to blocking AST (req@ffff928431be9b00 x1623662782419760 status 0 rc -110), evict it ns: filter-fir-OST001d_UUID lock: ffff92639d944a40/0x4162c98b034e1a52 lrc: 4/0,0 mode: PR/PR res: [0x2a973:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c29ccc9 expref: 42 pid: 64300 timeout: 1639898 lvb_type: 1 [1639794.156110] LustreError: 138-a: fir-OST001d: A client on nid 10.9.0.2@o2ib4 was evicted due to a lock blocking callback time out: rc -110 [1639794.156111] LustreError: Skipped 6 previous similar messages [1639794.156130] LustreError: 63402:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 154s: evicting client at 10.9.0.2@o2ib4 ns: filter-fir-OST001d_UUID lock: ffff925ee24b33c0/0x4162c98b034e1a36 lrc: 3/0,0 mode: PR/PR res: [0x2a969:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 131072->16777215) flags: 0x60000400010020 nid: 10.9.0.2@o2ib4 remote: 0x5d2fc5668c29cc7c expref: 43 pid: 64381 timeout: 0 lvb_type: 1 [1639794.156132] LustreError: 63402:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 5 previous similar messages [1639794.156522] LustreError: 69449:0:(client.c:1175:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@ffff9283ef2c3c00 x1623662782633440/t0(0) o104->fir-OST001d@10.9.0.2@o2ib4:15/16 lens 296/224 e 0 to 0 dl 0 ref 1 fl Rpc:/0/ffffffff rc 0/-1 [1639794.316779] Lustre: 69446:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 221 previous similar messages [1640102.908570] Lustre: fir-OST0021: Connection restored to 948ddf77-30e8-6f7f-29c3-d9d6fe8d8435 (at 10.9.101.4@o2ib4) [1640102.919101] Lustre: Skipped 62 previous similar messages [1640756.248141] Lustre: fir-OST0019: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1640756.258672] Lustre: Skipped 52 previous similar messages [1641711.516914] Lustre: fir-OST001d: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1641711.527444] Lustre: Skipped 43 previous similar messages [1642395.111850] Lustre: fir-OST0019: Connection restored to dba0ac03-f756-7e65-44de-ab2d15018d1d (at 10.9.101.41@o2ib4) [1642395.122463] Lustre: Skipped 24 previous similar messages [1643127.326061] Lustre: fir-OST001d: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1643127.336586] Lustre: Skipped 23 previous similar messages [1643838.795577] Lustre: fir-OST0019: Connection restored to 0732bf9f-f5f9-3f7f-d05c-6eafb128d46c (at 10.8.0.66@o2ib6) [1643838.806168] Lustre: Skipped 37 previous similar messages [1644466.814264] Lustre: fir-OST0019: Connection restored to dfbc3ca1-e3f2-2f8c-ef1d-6d31c339e5cc (at 10.9.103.31@o2ib4) [1644466.824881] Lustre: Skipped 8 previous similar messages [1645107.345003] Lustre: fir-OST0019: Connection restored to c46b7f3d-ec55-7fd7-e207-e9c6f5525b60 (at 10.8.0.68@o2ib6) [1645107.355437] Lustre: Skipped 31 previous similar messages [1645737.001099] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1645737.008417] Lustre: Skipped 24 previous similar messages [1646339.595174] Lustre: fir-OST001b: Connection restored to (at 10.0.10.3@o2ib7) [1646339.602488] Lustre: Skipped 38 previous similar messages [1646948.364313] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1646948.371624] Lustre: Skipped 43 previous similar messages [1647578.158250] Lustre: fir-OST001d: Connection restored to (at 10.0.10.3@o2ib7) [1647578.165563] Lustre: Skipped 36 previous similar messages [1648184.865586] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1648184.872897] Lustre: Skipped 23 previous similar messages [1648814.531827] Lustre: fir-OST0019: Connection restored to d295bc3f-a6cd-b523-283f-d68b438ddc06 (at 10.8.3.13@o2ib6) [1648814.542260] Lustre: Skipped 36 previous similar messages [1649861.932254] Lustre: fir-OST0019: Connection restored to 0732bf9f-f5f9-3f7f-d05c-6eafb128d46c (at 10.8.0.66@o2ib6) [1649861.942692] Lustre: Skipped 36 previous similar messages [1650681.997735] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1650682.008086] Lustre: Skipped 28 previous similar messages [1651762.990069] Lustre: fir-OST001b: Connection restored to 55a5f43f-2070-1278-9f4b-9a1845c8940d (at 10.8.11.22@o2ib6) [1651763.000596] Lustre: Skipped 5 previous similar messages [1653287.298735] Lustre: fir-OST001b: Connection restored to 55a5f43f-2070-1278-9f4b-9a1845c8940d (at 10.8.11.22@o2ib6) [1653287.309259] Lustre: Skipped 11 previous similar messages [1654034.093664] Lustre: fir-OST001b: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1654034.104098] Lustre: Skipped 5 previous similar messages [1654311.711731] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1654357.713491] Lustre: fir-OST0019: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1654357.723846] Lustre: Skipped 12 previous similar messages [1654401.177784] Lustre: fir-OST001f: Connection restored to 3577bb83-18fa-14e9-4b08-b3335e8f2568 (at 10.9.0.1@o2ib4) [1654401.188136] Lustre: Skipped 8 previous similar messages [1654482.986286] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1654482.993597] Lustre: Skipped 6 previous similar messages [1654677.605896] Lustre: fir-OST0019: Connection restored to 98df58b1-8a0c-b210-852f-29bcfcaceadc (at 10.9.103.40@o2ib4) [1654677.616513] Lustre: Skipped 11 previous similar messages [1654979.049649] Lustre: fir-OST0019: Connection restored to 1e03043d-571f-f709-9076-72fb7d056ac3 (at 10.9.103.43@o2ib4) [1654979.060259] Lustre: Skipped 73 previous similar messages [1655593.697793] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1655593.705122] Lustre: Skipped 183 previous similar messages [1656193.715932] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1656193.723276] Lustre: Skipped 20 previous similar messages [1656795.541404] Lustre: fir-OST0019: Connection restored to 5daab973-4b80-b2b7-c9e1-b8d5b6fc11e2 (at 10.9.105.2@o2ib4) [1656795.551931] Lustre: Skipped 1002 previous similar messages [1657532.787086] Lustre: fir-OST0019: Connection restored to 7cba68d6-88e2-7226-0b4f-cb83c3107f8f (at 10.9.102.34@o2ib4) [1657532.797696] Lustre: Skipped 47 previous similar messages [1658230.138762] Lustre: fir-OST0019: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1658230.149196] Lustre: Skipped 251 previous similar messages [1658837.431218] Lustre: fir-OST0019: Connection restored to 480f6595-8ac1-2fa1-010d-61ed12a06db6 (at 10.10.112.13@tcp) [1658837.441754] Lustre: Skipped 110 previous similar messages [1659493.822440] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1659493.829752] Lustre: Skipped 168 previous similar messages [1660094.835658] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1660094.842971] Lustre: Skipped 11 previous similar messages [1660694.852491] Lustre: fir-OST0021: Connection restored to (at 10.0.10.3@o2ib7) [1660694.859817] Lustre: Skipped 21 previous similar messages [1661358.919983] Lustre: fir-OST0019: Connection restored to 8ae4cc68-204d-d2bb-4908-94f49692744b (at 10.8.20.15@o2ib6) [1661358.930504] Lustre: Skipped 18 previous similar messages [1661960.303994] Lustre: fir-OST0019: Connection restored to 0e25277b-1ad0-ec5f-2777-56d7cafdcd31 (at 10.8.17.6@o2ib6) [1661960.314429] Lustre: Skipped 345 previous similar messages [1662618.539026] Lustre: fir-OST0019: Connection restored to e4e1992b-645c-7fa0-95fb-f97f854d138b (at 10.8.26.23@o2ib6) [1662618.549550] Lustre: Skipped 729 previous similar messages [1663394.848536] Lustre: fir-OST0019: Connection restored to (at 10.0.10.3@o2ib7) [1663394.855853] Lustre: Skipped 12 previous similar messages [1663997.958853] Lustre: fir-OST0019: Connection restored to b34d8462-79a8-9e1d-486f-37c6d2a9f0f0 (at 10.9.101.37@o2ib4) [1663997.969473] Lustre: Skipped 966 previous similar messages [1664601.188306] Lustre: fir-OST0019: Connection restored to 2df312f9-aff0-54ca-41b5-f915ec943155 (at 10.8.11.9@o2ib6) [1664601.198744] Lustre: Skipped 277 previous similar messages [1665234.766643] Lustre: fir-OST001b: Connection restored to 084ae2e7-32eb-f718-134b-ac7a3c2328d4 (at 10.9.101.9@o2ib4) [1665234.777171] Lustre: Skipped 134 previous similar messages [1665359.846746] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665359.857549] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443022/real 1549443022] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443033 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [1665359.885333] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7627 previous similar messages [1665360.347392] LNetError: 118485:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665360.358266] LNetError: 118485:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 104624 previous similar messages [1665361.347416] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665361.358196] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 203815 previous similar messages [1665363.347466] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665363.358248] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 416148 previous similar messages [1665367.347566] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665367.358358] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 833098 previous similar messages [1665375.347741] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665375.358530] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 1661093 previous similar messages [1665375.857760] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443038/real 1549443038] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443049 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665375.885553] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3318210 previous similar messages [1665387.482500] Lustre: fir-OST0019: Export ffff928479f2cc00 already connecting from 10.8.8.22@o2ib6 [1665391.348116] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665391.358896] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 3334582 previous similar messages [1665391.871926] Lustre: 12892:0:(service.c:2165:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (6:2s); client may timeout. req@ffff9273b3e99050 x1624702166607072/t0(0) o9->41732ea7-65b1-83d7-b5d1-efb7145a422b@10.9.112.4@o2ib4:342/0 lens 224/192 e 0 to 0 dl 1549443052 ref 1 fl Complete:/0/0 rc 0/0 [1665391.900133] Lustre: 12892:0:(service.c:2165:ptlrpc_server_handle_request()) Skipped 25 previous similar messages [1665403.878924] Lustre: 72779:0:(service.c:2165:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (6:2s); client may timeout. req@ffff9273b3e9e850 x1624697955880240/t0(0) o9->079c8b1d-b2a1-b068-d381-29e508dca751@10.9.103.22@o2ib4:354/0 lens 224/192 e 0 to 0 dl 1549443064 ref 1 fl Complete:/0/0 rc 0/0 [1665403.907217] Lustre: 72779:0:(service.c:2165:ptlrpc_server_handle_request()) Skipped 2 previous similar messages [1665407.858490] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443070/real 1549443070] req@ffff926463bcc200 x1623662806587104/t0(0) o106->fir-OST001d@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443081 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665407.886343] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6683066 previous similar messages [1665407.924077] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.103.9@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665412.527719] LustreError: 137-5: fir-OST0020_UUID: not available for connect from 10.8.8.22@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665415.039475] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.104.49@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665423.348849] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665423.359628] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 6740241 previous similar messages [1665429.091934] LustreError: 137-5: fir-OST0022_UUID: not available for connect from 10.9.104.45@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665437.615659] Lustre: fir-OST0019: Client ede7e3bd-7369-7fe3-3298-1a949c599650 (at 10.8.8.22@o2ib6) reconnecting [1665455.910698] Lustre: fir-OST0021: Client ce970d8c-cf83-fbcf-a0f2-5b0ed3cb9f89 (at 10.9.104.43@o2ib4) reconnecting [1665455.921050] Lustre: Skipped 1 previous similar message [1665458.101397] LustreError: 137-5: fir-OST0020_UUID: not available for connect from 10.9.103.9@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665471.859975] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443134/real 1549443134] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443145 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665471.887766] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 13403449 previous similar messages [1665487.350335] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665487.361112] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 13335986 previous similar messages [1665488.758553] Lustre: fir-OST0019: Client 029e2f9e-d883-e8d0-d1e8-9a1826095341 (at 10.8.30.33@o2ib6) reconnecting [1665488.768826] Lustre: Skipped 3 previous similar messages [1665529.446310] Lustre: fir-OST001d: Client 4712ded9-305f-102d-4761-daabc1364d31 (at 10.9.104.45@o2ib4) reconnecting [1665529.456658] Lustre: Skipped 2 previous similar messages [1665559.905010] LNet: Service thread pid 79216 was inactive for 200.04s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1665559.922119] Pid: 79216, comm: ll_ost00_006 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665559.932333] Call Trace: [1665559.934962] [] 0xffffffffffffffff [1665559.940172] LustreError: dumping log to /tmp/lustre-log.1549443222.79216 [1665561.193062] LNet: Service thread pid 64300 was inactive for 201.33s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1665561.210166] Pid: 64300, comm: ll_ost00_000 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665561.220334] Call Trace: [1665561.222969] [] 0xffffffffffffffff [1665561.228176] Pid: 118485, comm: ll_ost00_010 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665561.238446] Call Trace: [1665561.241073] [] 0xffffffffffffffff [1665561.246246] Pid: 64303, comm: ll_ost00_002 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1665561.256428] Call Trace: [1665561.259056] [] 0xffffffffffffffff [1665599.862935] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443262/real 1549443262] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443273 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665599.890708] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 26679244 previous similar messages [1665615.353284] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665615.364068] LNetError: 64303:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 26696085 previous similar messages [1665760.171611] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.9.105.17@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1665854.897389] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1665854.904703] Lustre: Skipped 183 previous similar messages [1665855.868843] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549443518/real 1549443518] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549443529 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1665855.896616] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 53397601 previous similar messages [1665871.359194] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1665871.369971] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 53424247 previous similar messages [1665954.522130] Lustre: 61176:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff92647cfce050 x1623967095657056/t0(0) o101->0ccdc4e2-9749-c9a5-afb4-85874ce74d6c@10.0.10.3@o2ib7:157/0 lens 336/400 e 24 to 0 dl 1549443622 ref 2 fl Interpret:/0/0 rc 0/0 [1665960.437869] Lustre: fir-OST001d: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1666366.186428] Lustre: fir-OST001d: Client f9ba6989-1b37-0945-75ce-916db80d0755 (at 10.9.106.29@o2ib4) reconnecting [1666366.196774] Lustre: Skipped 4 previous similar messages [1666367.880659] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549444030/real 1549444030] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549444041 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1666367.908428] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 106869708 previous similar messages [1666383.371016] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1666383.381794] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 106835317 previous similar messages [1666485.859430] Lustre: fir-OST0023: Connection restored to f05f50bb-5878-44e8-e806-fa2968d0f6ca (at 10.9.105.1@o2ib4) [1666485.869948] Lustre: Skipped 77 previous similar messages [1666708.159384] Lustre: fir-OST001f: Client 121431d1-d11c-6cf2-7fd0-eaf8c3ca6b3e (at 10.9.103.12@o2ib4) reconnecting [1666708.169733] Lustre: Skipped 7 previous similar messages [1666952.920187] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.8.22@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1666952.937553] LustreError: Skipped 5 previous similar messages [1666967.894525] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549444630/real 1549444630] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549444641 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1666967.922311] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 124717666 previous similar messages [1666983.384871] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1666983.395649] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 124676795 previous similar messages [1667086.358882] Lustre: fir-OST001b: Connection restored to c95ae3a3-cc0b-6e03-007e-3f43096cb7c1 (at 10.9.101.21@o2ib4) [1667086.369488] Lustre: Skipped 124 previous similar messages [1667148.868737] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.102.47@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1667487.723249] Lustre: fir-OST001f: Client 7936d72a-098c-6dfd-6893-32b658b77868 (at 10.9.103.9@o2ib4) reconnecting [1667487.733520] Lustre: Skipped 17 previous similar messages [1667567.908373] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549445230/real 1549445230] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549445241 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1667567.936142] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 125538991 previous similar messages [1667583.398727] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1667583.409504] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 125624767 previous similar messages [1667694.529244] Lustre: fir-OST001d: Connection restored to 64c2e10f-595e-bb8e-efbd-0544992f523c (at 10.9.101.49@o2ib4) [1667694.539858] Lustre: Skipped 135 previous similar messages [1667703.655765] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.8.13.15@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1667731.444715] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.102.38@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1667794.544225] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.8.18.19@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1667870.883705] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.103.5@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1667870.901158] LustreError: Skipped 1 previous similar message [1668062.082291] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.8.4.7@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668167.922225] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549445830/real 1549445830] req@ffff926463bcc200 x1623662806587104/t0(0) o106->fir-OST001d@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549445841 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1668167.950136] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 128035326 previous similar messages [1668173.283845] Lustre: fir-OST0019: Client 20cb63a2-4619-420f-25f6-c09916b5cd24 (at 10.8.17.16@o2ib6) reconnecting [1668173.294109] Lustre: Skipped 12 previous similar messages [1668183.412581] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1668183.423362] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 128101093 previous similar messages [1668296.261303] Lustre: fir-OST001b: Connection restored to 7adbd302-8979-36d3-3dcd-fabe944f2fe9 (at 10.8.11.25@o2ib6) [1668296.271820] Lustre: Skipped 328 previous similar messages [1668299.878212] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.8.7.13@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1668299.895584] LustreError: Skipped 15 previous similar messages [1668537.154775] Lustre: fir-OST001b: Export ffff9274632fd400 already connecting from 10.8.8.22@o2ib6 [1668537.163732] Lustre: Skipped 1 previous similar message [1668568.776983] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.10.16.5@tcp (no target). If you are running an HA pair check that the target is mounted on the other server. [1668568.794263] LustreError: Skipped 21 previous similar messages [1668767.936079] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549446430/real 1549446430] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549446441 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1668767.963855] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 139816115 previous similar messages [1668783.426437] LNetError: 118485:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1668783.437302] LNetError: 118485:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 140368479 previous similar messages [1668786.643672] Lustre: fir-OST0019: Client 994c77ca-5a3e-accc-3ddc-a08f18403cd1 (at 10.8.17.23@o2ib6) reconnecting [1668786.653939] Lustre: Skipped 77 previous similar messages [1668901.255719] Lustre: fir-OST0019: Connection restored to a9d1952c-c35e-2d17-0ac7-abfee221f073 (at 10.8.9.2@o2ib6) [1668901.266070] Lustre: Skipped 871 previous similar messages [1669085.618825] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.11.4@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1669085.636196] LustreError: Skipped 34 previous similar messages [1669362.070470] Lustre: fir-OST001d: haven't heard from client b1ae38e6-4e9c-51ec-0d30-af325c63f3ef (at 10.8.4.15@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92644739e800, cur 1549447025 expire 1549446875 last 1549446798 [1669362.092255] Lustre: Skipped 3 previous similar messages [1669363.872882] Lustre: fir-OST001b: haven't heard from client b1ae38e6-4e9c-51ec-0d30-af325c63f3ef (at 10.8.4.15@o2ib6) in 228 seconds. I think it's dead, and I am evicting it. exp ffff927368fc2400, cur 1549447026 expire 1549446876 last 1549446798 [1669367.949934] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549447030/real 1549447030] req@ffff926463bcc200 x1623662806587104/t0(0) o106->fir-OST001d@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549447041 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1669367.977792] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 148525425 previous similar messages [1669383.440288] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1669383.451063] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 148475492 previous similar messages [1669509.340930] Lustre: fir-OST001b: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1669509.351459] Lustre: Skipped 407 previous similar messages [1669567.332913] Lustre: fir-OST001d: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1669567.343098] Lustre: Skipped 37 previous similar messages [1669818.457300] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.104.55@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1669818.474839] LustreError: Skipped 23 previous similar messages [1669967.963784] Lustre: 64300:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549447630/real 1549447630] req@ffff926137bd3300 x1623662806587088/t0(0) o106->fir-OST0021@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549447641 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1669967.991554] Lustre: 64300:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 147915672 previous similar messages [1669983.454141] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1669983.464924] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 147926621 previous similar messages [1670115.756611] Lustre: fir-OST0021: Connection restored to 5b8aac1d-db3a-957a-49fa-fd5d736eb555 (at 10.8.27.13@o2ib6) [1670115.767179] Lustre: Skipped 361 previous similar messages [1670168.307528] Lustre: fir-OST001d: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1670168.317701] Lustre: Skipped 21 previous similar messages [1670427.318331] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.8.24.11@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1670427.335787] LustreError: Skipped 18 previous similar messages [1670567.977641] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549448230/real 1549448230] req@ffff9256bf2a1800 x1623662806587056/t0(0) o106->fir-OST0019@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549448241 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1670568.005478] Lustre: 64303:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 147357203 previous similar messages [1670583.467997] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1670583.478775] LNetError: 79216:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 147340715 previous similar messages [1670721.251118] Lustre: fir-OST0021: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1670721.261641] Lustre: Skipped 127 previous similar messages [1670769.283253] Lustre: fir-OST001d: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1670769.293428] Lustre: Skipped 5 previous similar messages [1671167.991493] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549448830/real 1549448830] req@ffff926463bcc200 x1623662806587104/t0(0) o106->fir-OST001d@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549448841 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1671168.019360] Lustre: 118485:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 144858625 previous similar messages [1671183.481850] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.114.10@tcp from [1671183.492629] LNetError: 64300:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 144403284 previous similar messages [1671298.869451] LNet: Service thread pid 79216 completed after 5938.87s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [1671298.869453] LNet: Service thread pid 64303 completed after 5938.88s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [1671298.869478] LustreError: 64300:0:(client.c:1175:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@ffff926137bd3300 x1623662806587088/t0(0) o106->fir-OST0021@10.10.114.10@tcp:15/16 lens 296/280 e 0 to 1 dl 1549448972 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1671298.924440] LNet: Skipped 2 previous similar messages [1671355.190199] Lustre: fir-OST001b: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1671355.200725] Lustre: Skipped 88 previous similar messages [1672027.635628] Lustre: fir-OST001b: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1672027.646160] Lustre: Skipped 59 previous similar messages [1672648.879915] Lustre: fir-OST001f: Connection restored to 76e809db-e90a-7c07-20d4-e3130ed3be85 (at 10.9.104.30@o2ib4) [1672648.890529] Lustre: Skipped 146 previous similar messages [1673262.973485] Lustre: fir-OST0023: Connection restored to 8d1694af-d32c-3288-00b1-fe38a72175fb (at 10.9.104.37@o2ib4) [1673262.984108] Lustre: Skipped 62 previous similar messages [1673863.209426] Lustre: fir-OST001b: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1673863.219954] Lustre: Skipped 141 previous similar messages [1674471.146803] Lustre: fir-OST0019: Connection restored to 7a28562b-3b5d-09cc-c2f6-568559bac302 (at 10.8.18.15@o2ib6) [1674471.157331] Lustre: Skipped 115 previous similar messages [1675075.282510] Lustre: fir-OST0021: Connection restored to e78a174a-ae69-cb32-9b42-0306c7153992 (at 10.9.103.7@o2ib4) [1675075.293030] Lustre: Skipped 214 previous similar messages [1675677.682099] Lustre: fir-OST0019: Connection restored to b41950de-0614-6c1a-0d53-d43c60fe0f33 (at 10.9.102.1@o2ib4) [1675677.692699] Lustre: Skipped 27 previous similar messages [1676294.222222] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1676294.229541] Lustre: Skipped 32 previous similar messages [1676923.479952] Lustre: fir-OST001d: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1676923.490473] Lustre: Skipped 40 previous similar messages [1677525.248139] Lustre: fir-OST0021: Connection restored to ee310e5c-55e4-0a0f-af99-ae0536891531 (at 10.8.27.2@o2ib6) [1677525.258574] Lustre: Skipped 26 previous similar messages [1678154.030655] Lustre: fir-OST001f: Connection restored to ee310e5c-55e4-0a0f-af99-ae0536891531 (at 10.8.27.2@o2ib6) [1678154.041108] Lustre: Skipped 58 previous similar messages [1678823.372391] Lustre: fir-OST0019: Connection restored to 7cba68d6-88e2-7226-0b4f-cb83c3107f8f (at 10.9.102.34@o2ib4) [1678823.383003] Lustre: Skipped 30 previous similar messages [1679429.916153] Lustre: fir-OST001d: Connection restored to 9114d135-37ef-ec55-8f47-397cb880c459 (at 10.8.28.6@o2ib6) [1679429.926588] Lustre: Skipped 28 previous similar messages [1680094.759423] Lustre: fir-OST0023: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1680094.769969] Lustre: Skipped 35 previous similar messages [1680695.925606] Lustre: fir-OST001b: Connection restored to 7a5c5a07-538c-91c6-9b36-9ce68e18d76c (at 10.9.105.59@o2ib4) [1680695.936212] Lustre: Skipped 60 previous similar messages [1681300.656016] Lustre: fir-OST0019: Connection restored to 2cc850be-dc6d-45a9-7198-5a65d8b1618d (at 10.9.104.39@o2ib4) [1681300.666628] Lustre: Skipped 48 previous similar messages [1681901.521361] Lustre: fir-OST0019: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1681901.531880] Lustre: Skipped 27 previous similar messages [1682526.837339] Lustre: fir-OST0023: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1682526.847870] Lustre: Skipped 23 previous similar messages [1683142.757526] Lustre: fir-OST0021: Connection restored to 9f0fc7f1-3017-a377-d5ca-3f45fac7d96c (at 10.9.106.2@o2ib4) [1683142.768069] Lustre: Skipped 51 previous similar messages [1683794.219067] Lustre: fir-OST0019: Connection restored to 59de79d8-9608-a209-2703-a97ba62324b7 (at 10.8.18.19@o2ib6) [1683794.229583] Lustre: Skipped 32 previous similar messages [1684394.406678] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1684394.414020] Lustre: Skipped 35 previous similar messages [1684994.422929] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1684994.430244] Lustre: Skipped 26 previous similar messages [1685594.436943] Lustre: fir-OST0023: Connection restored to (at 10.0.10.3@o2ib7) [1685594.444262] Lustre: Skipped 387 previous similar messages [1686194.454575] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1686194.461889] Lustre: Skipped 280 previous similar messages [1686570.441699] Lustre: fir-OST0023: haven't heard from client ef658cd6-a6d2-3958-b89c-2c53bab505a2 (at 10.8.27.35@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92747b487800, cur 1549464233 expire 1549464083 last 1549464006 [1686570.463663] Lustre: Skipped 4 previous similar messages [1686794.471198] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1686794.478510] Lustre: Skipped 209 previous similar messages [1686875.447061] Lustre: fir-OST001d: haven't heard from client ee310e5c-55e4-0a0f-af99-ae0536891531 (at 10.8.27.2@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92747bdaac00, cur 1549464538 expire 1549464388 last 1549464311 [1686875.468852] Lustre: Skipped 5 previous similar messages [1687394.486855] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1687394.494177] Lustre: Skipped 362 previous similar messages [1687994.501272] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1687994.508591] Lustre: Skipped 171 previous similar messages [1688594.517731] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1688594.525042] Lustre: Skipped 314 previous similar messages [1689194.534021] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1689194.541339] Lustre: Skipped 152 previous similar messages [1689794.548882] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1689794.556198] Lustre: Skipped 111 previous similar messages [1690083.413264] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1690083.423437] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Skipped 7 previous similar messages [1690083.433777] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (6): c: 0, oc: 0, rc: 8 [1690083.445848] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Skipped 7 previous similar messages [1690083.456298] Lustre: 61947:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549467740/real 1549467745] req@ffff927c928e5d00 x1623662812932944/t0(0) o400->fir-MDT0002-lwp-OST001f@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549467972 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1690083.456334] Lustre: fir-MDT0002-lwp-OST001b: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1690083.456336] Lustre: Skipped 2 previous similar messages [1690083.456389] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1690083.520748] Lustre: 61947:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 28329890 previous similar messages [1690084.413295] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [1690084.423619] Lustre: fir-MDT0000-lwp-OST0019: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1690084.439797] Lustre: Skipped 10 previous similar messages [1690087.413354] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 4 seconds [1690087.423608] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 14 previous similar messages [1690090.413429] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1690090.423688] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1690135.414467] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 5 seconds [1690190.415746] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1690190.425924] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (6): c: 0, oc: 0, rc: 8 [1690223.174126] Lustre: fir-OST001b: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1690223.183265] Lustre: Skipped 4 previous similar messages [1690242.020183] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1690242.020317] Lustre: fir-OST001f: deleting orphan objects from 0x0:677611 to 0x0:677633 [1690242.020926] Lustre: fir-OST0021: deleting orphan objects from 0x0:677593 to 0x0:677633 [1690242.021308] Lustre: fir-OST001d: deleting orphan objects from 0x0:677642 to 0x0:677665 [1690242.021809] Lustre: fir-OST001b: deleting orphan objects from 0x0:677665 to 0x0:677697 [1690242.021923] Lustre: fir-OST0023: deleting orphan objects from 0x0:677604 to 0x0:677633 [1690242.022296] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:472276 to 0x480000400:472449 [1690242.022426] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:472317 to 0x640000400:472353 [1690242.022648] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:472272 to 0x980000402:472353 [1690242.022837] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:472210 to 0xcc0000402:472289 [1690242.023019] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:472237 to 0xd40000402:472289 [1690242.125604] LustreError: Skipped 12 previous similar messages [1690267.109354] Lustre: fir-OST0019: deleting orphan objects from 0x0:677587 to 0x0:677633 [1690267.117642] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:472262 to 0xd00000402:472417 [1690395.567234] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1690395.574552] Lustre: Skipped 139 previous similar messages [1690984.556174] LNetError: 46588:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.15@tcp from [1690984.566968] LNetError: 46588:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 24996306 previous similar messages [1690984.578226] Lustre: 46588:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468647/real 1549468647] req@ffff928431be8900 x1623662813677920/t0(0) o104->fir-OST001b@10.10.113.15@tcp:15/16 lens 296/224 e 0 to 1 dl 1549468658 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [1690984.606005] Lustre: 46588:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5447 previous similar messages [1690993.953285] Lustre: 69622:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468656/real 1549468656] req@ffff927eb6e06600 x1623662813678048/t0(0) o104->fir-OST001d@10.10.113.15@tcp:15/16 lens 296/224 e 0 to 1 dl 1549468667 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1690993.953287] Lustre: 46588:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468656/real 1549468656] req@ffff928431be8900 x1623662813677920/t0(0) o104->fir-OST001b@10.10.113.15@tcp:15/16 lens 296/224 e 0 to 1 dl 1549468667 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1690993.953291] Lustre: 46588:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 2629725 previous similar messages [1690994.019261] Lustre: 69622:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 12804 previous similar messages [1691000.017180] Lustre: fir-OST0023: Connection restored to e391b7dc-837c-aad7-6ebd-5ea1c73131db (at 10.8.18.24@o2ib6) [1691000.027698] Lustre: Skipped 154 previous similar messages [1691031.454151] Lustre: 46588:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468693/real 1549468693] req@ffff928431be8900 x1623662813677920/t0(0) o104->fir-OST001b@10.10.113.15@tcp:15/16 lens 296/224 e 0 to 1 dl 1549468704 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1691031.481919] Lustre: 46588:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 10604521 previous similar messages [1691059.558797] LNetError: 118099:0:(lib-move.c:1980:lnet_handle_find_routed_path()) no route to 10.10.113.15@tcp from [1691059.569662] LNetError: 118099:0:(lib-move.c:1980:lnet_handle_find_routed_path()) Skipped 21158617 previous similar messages [1691067.076860] LustreError: 137-5: fir-OST0020_UUID: not available for connect from 10.8.7.35@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1691070.789163] Lustre: fir-OST0021: Client 5b322ed5-1f83-3d4a-541c-ca479ed4d108 (at 10.8.18.18@o2ib6) reconnecting [1691070.799426] Lustre: Skipped 12 previous similar messages [1691083.187777] Lustre: fir-OST0023: Client 0ccdc4e2-9749-c9a5-afb4-85874ce74d6c (at 10.0.10.3@o2ib7) reconnecting [1691106.455877] Lustre: 118099:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549468768/real 1549468768] req@ffff927c928e3600 x1623662813678064/t0(0) o104->fir-OST001f@10.10.113.15@tcp:15/16 lens 296/224 e 0 to 1 dl 1549468779 ref 1 fl Rpc:eX/2/ffffffff rc 0/-1 [1691106.483736] Lustre: 118099:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 21150154 previous similar messages [1691134.888554] LustreError: 46588:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) ### client (nid 10.10.113.15@tcp) failed to reply to blocking AST (req@ffff928431be8900 x1623662813677920 status 0 rc -110), evict it ns: filter-fir-OST001b_UUID lock: ffff927f776606c0/0x4162c98b056c35b2 lrc: 4/0,0 mode: PR/PR res: [0xd00000402:0x72c38:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 4294967296->34359738367) flags: 0x60000400010020 nid: 10.10.113.15@tcp remote: 0xdf1b55af52e166c4 expref: 6 pid: 92984 timeout: 1691244 lvb_type: 1 [1691134.888557] LustreError: 69622:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) ### client (nid 10.10.113.15@tcp) failed to reply to blocking AST (req@ffff927eb6e06600 x1623662813678048 status 0 rc -110), evict it ns: filter-fir-OST001d_UUID lock: ffff927042698d80/0x4162c98b056c35a4 lrc: 4/0,0 mode: PR/PR res: [0xcc0000402:0x72c1c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 8388608->536870911) flags: 0x60000400010020 nid: 10.10.113.15@tcp remote: 0xdf1b55af52e1668c expref: 6 pid: 7568 timeout: 1691244 lvb_type: 1 [1691134.888561] LustreError: 69622:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) Skipped 4 previous similar messages [1691134.888563] LustreError: 138-a: fir-OST0023: A client on nid 10.10.113.15@tcp was evicted due to a lock blocking callback time out: rc -110 [1691134.888565] LustreError: Skipped 4 previous similar messages [1691134.888586] LustreError: 63402:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 150s: evicting client at 10.10.113.15@tcp ns: filter-fir-OST0023_UUID lock: ffff9293d1b59b00/0x4162c98b056c35c0 lrc: 3/0,0 mode: PR/PR res: [0xd40000402:0x72c20:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 17179869184->18446744073709551615) flags: 0x60000400010020 nid: 10.10.113.15@tcp remote: 0xdf1b55af52e166fc expref: 7 pid: 64381 timeout: 0 lvb_type: 1 [1691134.888588] LustreError: 63402:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 2 previous similar messages [1691135.062629] LustreError: 46588:0:(ldlm_lockd.c:682:ldlm_handle_ast_error()) Skipped 1 previous similar message [1691135.480050] LustreError: 63402:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 150s: evicting client at 10.10.113.15@tcp ns: filter-fir-OST001f_UUID lock: ffff9293d1b5de80/0x4162c98b056c35c7 lrc: 3/0,0 mode: PR/PR res: [0x480000400:0x72c4a:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 17179869184->18446744073709551615) flags: 0x60000400010020 nid: 10.10.113.15@tcp remote: 0xdf1b55af52e16734 expref: 7 pid: 64381 timeout: 0 lvb_type: 1 [1691135.522338] LustreError: 63402:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 1 previous similar message [1691151.778325] LustreError: 137-5: fir-OST0020_UUID: not available for connect from 10.8.31.3@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1691152.610513] Lustre: fir-OST001d: Client a1a32c6e-a603-c7ad-bbfa-4137583a5bae (at 10.8.17.20@o2ib6) reconnecting [1691152.620808] Lustre: Skipped 1 previous similar message [1691600.107064] Lustre: fir-OST001f: Connection restored to (at 10.0.10.3@o2ib7) [1691600.114382] Lustre: Skipped 643 previous similar messages [1691844.321978] Lustre: fir-OST001f: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1691844.598297] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:476010 to 0x480000400:476033 [1691884.638458] Lustre: fir-OST001d: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1691884.647598] Lustre: Skipped 6 previous similar messages [1691885.998232] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:475914 to 0x640000400:476001 [1691885.998324] Lustre: fir-OST0023: deleting orphan objects from 0x0:677645 to 0x0:677665 [1691885.998326] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:475846 to 0xd40000402:475937 [1691885.998497] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:475843 to 0xcc0000402:475937 [1691893.440051] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:475914 to 0x640000400:476033 [1691899.221349] Lustre: fir-OST001d: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1691899.230495] Lustre: Skipped 1 previous similar message [1691899.238856] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:475843 to 0xcc0000402:475969 [1691924.252582] Lustre: fir-OST001b: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1691924.404070] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:475987 to 0xd00000402:476065 [1691961.892765] Lustre: fir-OST001f: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1691965.400538] Lustre: fir-OST001f: deleting orphan objects from 0x0:677645 to 0x0:677665 [1691969.999307] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:475987 to 0xd00000402:476097 [1692003.910838] LustreError: 137-5: fir-OST0020_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1692003.928321] LustreError: Skipped 4 previous similar messages [1692005.012985] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:475987 to 0xd00000402:476129 [1692028.953702] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1692028.971161] LustreError: Skipped 6 previous similar messages [1692204.198328] Lustre: fir-OST0023: Connection restored to 3a10c79f-70d6-d387-a9b4-ab231a26d59b (at 10.9.102.24@o2ib4) [1692204.208940] Lustre: Skipped 660 previous similar messages [1692804.244154] Lustre: fir-OST0023: Connection restored to 911cece2-4daf-122d-bab9-412e767c25bb (at 10.8.18.23@o2ib6) [1692804.254680] Lustre: Skipped 559 previous similar messages [1692925.461755] Lustre: fir-OST001d: deleting orphan objects from 0x0:677676 to 0x0:677697 [1692925.461765] Lustre: fir-OST001b: deleting orphan objects from 0x0:677709 to 0x0:677729 [1693225.805659] Lustre: fir-OST0021: deleting orphan objects from 0x0:677645 to 0x0:677665 [1693406.039093] Lustre: fir-OST001d: Connection restored to 3a10c79f-70d6-d387-a9b4-ab231a26d59b (at 10.9.102.24@o2ib4) [1693406.049732] Lustre: Skipped 156 previous similar messages [1694007.182375] Lustre: fir-OST001d: Connection restored to ea7067de-16a0-5aef-a5e8-0c3de073c862 (at 10.9.106.72@o2ib4) [1694007.192988] Lustre: Skipped 254 previous similar messages [1694246.799725] Lustre: fir-OST001f: haven't heard from client 40ad8941-e56d-16a8-06a4-9f03d85f8e6b (at 10.9.107.64@o2ib4) in 227 seconds. I think it's dead, and I am evicting it. exp ffff926de0a46800, cur 1549471909 expire 1549471759 last 1549471682 [1694246.821711] Lustre: Skipped 2 previous similar messages [1694611.776660] Lustre: fir-OST001d: Connection restored to dc527ec4-db62-14c0-459e-fcb26c78037c (at 10.9.107.8@o2ib4) [1694611.787188] Lustre: Skipped 313 previous similar messages [1695219.122623] Lustre: fir-OST001b: Connection restored to 1127e5ab-6d36-a097-38c2-cf0f21f5ec17 (at 10.8.26.8@o2ib6) [1695219.133055] Lustre: Skipped 368 previous similar messages [1695572.175424] Lustre: fir-OST0023: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1695572.184562] Lustre: Skipped 3 previous similar messages [1695572.261007] Lustre: 64305:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549473227/real 1549473227] req@ffff92682fc4b300 x1623662816178736/t0(0) o104->fir-OST0023@10.9.107.9@o2ib4:15/16 lens 296/224 e 0 to 1 dl 1549473234 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1695572.288425] Lustre: 64305:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 8023956 previous similar messages [1695574.048469] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:134396 to 0x980000401:134433 [1695580.384843] Lustre: fir-OST0023: deleting orphan objects from 0xd40000400:146757 to 0xd40000400:146817 [1695581.428240] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:146735 to 0xcc0000400:146817 [1695581.428639] Lustre: fir-OST0023: deleting orphan objects from 0xd40000400:146757 to 0xd40000400:146817 [1695581.785722] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:146661 to 0x480000401:146753 [1695584.170655] Lustre: fir-OST0021: Client fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1695584.179797] Lustre: Skipped 5 previous similar messages [1695584.187794] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:134396 to 0x980000401:134433 [1695599.540632] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1695599.550805] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.52@o2ib7 (5): c: 0, oc: 1, rc: 8 [1695601.806885] Lustre: fir-OST0019: deleting orphan objects from 0x640000401:146732 to 0x640000401:146817 [1695603.444145] Lustre: fir-OST001f: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1695603.453289] Lustre: Skipped 1 previous similar message [1695603.458765] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:146661 to 0x480000401:146785 [1695607.051207] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:146748 to 0x980000400:146785 [1695647.887752] Lustre: fir-OST001f: Client fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1695647.896887] Lustre: Skipped 1 previous similar message [1695649.028293] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:146794 to 0x480000401:146817 [1695684.732185] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.9.106.6@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1695684.749653] LustreError: Skipped 1 previous similar message [1695687.986217] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1695687.986710] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:146829 to 0xcc0000400:146849 [1695687.986871] Lustre: fir-OST001b: deleting orphan objects from 0xd00000400:146797 to 0xd00000400:146817 [1695687.986872] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:134453 to 0x480000402:134465 [1695687.986923] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:134438 to 0x980000401:134465 [1695687.987163] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:134470 to 0xd40000401:134497 [1695688.051073] LustreError: Skipped 7 previous similar messages [1695725.066143] Lustre: fir-OST0019: Client a37c418b-5455-262c-ca39-fe09a1d64b6a (at 10.9.106.7@o2ib4) reconnecting [1695725.076411] Lustre: Skipped 8 previous similar messages [1695738.528120] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1695738.545581] LustreError: Skipped 5 previous similar messages [1695741.381136] Lustre: fir-OST001b: deleting orphan objects from 0xd00000401:134352 to 0xd00000401:134369 [1695744.298446] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.9.106.7@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1695744.315905] LustreError: Skipped 1 previous similar message [1695751.984984] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:146858 to 0xcc0000400:146881 [1695751.984989] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:146831 to 0x480000401:146849 [1695751.984994] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:134470 to 0xd40000401:134529 [1695820.318876] Lustre: fir-OST001d: Connection restored to ef406a33-dbdc-c381-15c5-4fb662abecc1 (at 10.8.27.10@o2ib6) [1695820.329408] Lustre: Skipped 516 previous similar messages [1695831.477737] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:146862 to 0x480000401:146881 [1695832.532692] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:134486 to 0x980000401:134529 [1696328.883973] Lustre: fir-OST0023: Client fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1696328.893106] Lustre: Skipped 9 previous similar messages [1696332.484863] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:134618 to 0xcc0000401:134657 [1696336.745449] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:134724 to 0xd40000401:134753 [1696339.080008] Lustre: fir-OST001b: deleting orphan objects from 0xd00000401:134562 to 0xd00000401:134593 [1696339.080536] Lustre: fir-OST0019: deleting orphan objects from 0x640000401:146923 to 0x640000401:146945 [1696339.080631] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:134701 to 0x980000401:134721 [1696412.775152] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:146968 to 0xcc0000400:147009 [1696420.656617] Lustre: fir-OST001b: Connection restored to 7dac1966-a349-fc4a-4911-f2d61efbd1f7 (at 10.8.11.21@o2ib6) [1696420.667137] Lustre: Skipped 371 previous similar messages [1696423.948330] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1696444.535077] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1696444.552536] LustreError: Skipped 6 previous similar messages [1696454.193325] LustreError: 137-5: fir-OST0020_UUID: not available for connect from 10.0.10.52@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1696454.210790] LustreError: Skipped 3 previous similar messages [1696456.043136] Lustre: fir-OST0023: deleting orphan objects from 0xd40000400:146924 to 0xd40000400:146945 [1696456.043147] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:134701 to 0x980000401:134753 [1696456.519520] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:134618 to 0xcc0000401:134689 [1696456.535703] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:146955 to 0x480000401:146977 [1697025.294562] Lustre: fir-OST0019: Connection restored to e0d0d81e-4164-f236-7d8a-b466fb3eea50 (at 10.9.104.65@o2ib4) [1697025.305187] Lustre: Skipped 279 previous similar messages [1697626.409407] Lustre: fir-OST0021: Connection restored to 45f81bad-ba79-a5b8-97f4-c718cac35552 (at 10.8.0.82@o2ib6) [1697626.419860] Lustre: Skipped 378 previous similar messages [1698228.740244] Lustre: fir-OST001f: Connection restored to a82d0417-5354-7090-8c74-27f558bf90cb (at 10.9.103.27@o2ib4) [1698228.750859] Lustre: Skipped 338 previous similar messages [1698654.720614] Lustre: fir-OST001f: haven't heard from client 5fe1f35d-2720-6130-4811-b6c831015c6a (at 10.8.20.15@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff926a46ec3800, cur 1549476317 expire 1549476167 last 1549476090 [1698654.742518] Lustre: Skipped 1 previous similar message [1698829.352904] Lustre: fir-OST0023: Connection restored to 7e4f58d8-6e8c-32f9-59fb-7de9af0df4b4 (at 10.9.106.12@o2ib4) [1698829.363522] Lustre: Skipped 1022 previous similar messages [1698830.389663] Lustre: fir-OST001d: Client f97f4802-8c0b-e9cd-e8d7-93692decf22a (at 10.9.102.8@o2ib4) reconnecting [1698830.399924] Lustre: Skipped 15 previous similar messages [1698844.427992] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:147268 to 0x480000401:147297 [1698865.119400] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:135380 to 0x480000402:135425 [1698870.724144] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:147268 to 0x480000401:147329 [1698870.724264] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:147186 to 0x980000400:147201 [1698871.584854] Lustre: fir-OST001d: Client fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1698871.593991] Lustre: Skipped 4 previous similar messages [1698874.564593] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:135406 to 0xcc0000401:135425 [1698898.769299] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:147203 to 0x980000400:147233 [1698902.069142] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:135406 to 0xcc0000401:135457 [1698910.497206] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:147203 to 0x980000400:147265 [1699429.927974] Lustre: fir-OST0023: Connection restored to 09e96f09-88f3-17cc-ba26-88dee3b61d1c (at 10.8.7.12@o2ib6) [1699429.938414] Lustre: Skipped 1165 previous similar messages [1699467.773570] Lustre: fir-OST0021: haven't heard from client 57eeb417-c604-9e85-88ff-67a98be0814e (at 10.8.10.33@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9265135e7800, cur 1549477130 expire 1549476980 last 1549476903 [1699467.795481] Lustre: Skipped 5 previous similar messages [1699583.055065] Lustre: fir-OST001d: Client 7e4f58d8-6e8c-32f9-59fb-7de9af0df4b4 (at 10.9.106.12@o2ib4) reconnecting [1699583.065419] Lustre: Skipped 3 previous similar messages [1699620.108098] Lustre: fir-OST001b: Client 7e4f58d8-6e8c-32f9-59fb-7de9af0df4b4 (at 10.9.106.12@o2ib4) reconnecting [1699620.118457] Lustre: Skipped 2 previous similar messages [1699623.283552] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:135547 to 0x480000402:135585 [1699627.305277] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:147456 to 0x980000400:147489 [1699666.347924] Lustre: fir-OST001d: Client 7e4f58d8-6e8c-32f9-59fb-7de9af0df4b4 (at 10.9.106.12@o2ib4) reconnecting [1699666.358279] Lustre: Skipped 2 previous similar messages [1699689.362359] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.106.12@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1699689.379902] LustreError: Skipped 1 previous similar message [1699698.635287] LNetError: 61886:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1699698.645459] LNetError: 61886:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.52@o2ib7 (3): c: 0, oc: 3, rc: 8 [1699698.658057] Lustre: 61951:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549477355/real 1549477360] req@ffff927fee587500 x1623662818461280/t0(0) o400->fir-MDT0003-lwp-OST0019@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478112 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1699698.658110] Lustre: fir-MDT0001-lwp-OST0021: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1699698.703117] Lustre: 61951:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [1699699.635310] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1699699.645564] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 12 previous similar messages [1699699.655257] Lustre: fir-MDT0001-lwp-OST0019: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1699699.671438] Lustre: Skipped 7 previous similar messages [1699701.635355] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [1699701.645607] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 7 previous similar messages [1699706.635466] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1699706.645720] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1699707.833867] Lustre: fir-OST0021: Client fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1699707.843046] Lustre: Skipped 5 previous similar messages [1699707.853833] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:135587 to 0x980000401:135617 [1699708.891350] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:147526 to 0x480000401:147553 [1699759.000189] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:147498 to 0x980000400:147521 [1699759.000278] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:147502 to 0xcc0000400:147521 [1699759.000366] Lustre: fir-OST0019: deleting orphan objects from 0x640000401:147440 to 0x640000401:147457 [1699759.000418] Lustre: fir-OST001b: deleting orphan objects from 0xd00000400:147405 to 0xd00000400:147425 [1699759.000441] Lustre: fir-OST0023: deleting orphan objects from 0xd40000400:147439 to 0xd40000400:147457 [1699759.000542] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:135591 to 0x480000402:135617 [1699759.000643] Lustre: fir-OST0019: deleting orphan objects from 0x640000402:135419 to 0x640000402:135457 [1699759.001494] Lustre: fir-OST001b: deleting orphan objects from 0xd00000401:135441 to 0xd00000401:135457 [1699759.001540] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:135597 to 0xd40000401:135617 [1699759.001777] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:135584 to 0xcc0000401:135617 [1700033.437476] Lustre: fir-OST001f: Connection restored to e9d1b5f8-7ec9-998b-fe00-a6102cb74525 (at 10.9.102.2@o2ib4) [1700033.448007] Lustre: Skipped 1220 previous similar messages [1700134.752404] Lustre: fir-OST001f: haven't heard from client a13f0f5a-2377-84a5-980d-206962591c11 (at 10.8.11.10@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff927dd200d800, cur 1549477797 expire 1549477647 last 1549477570 [1700134.774291] Lustre: Skipped 1 previous similar message [1700450.142640] Lustre: 61957:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549477355/real 1549477355] req@ffff927fee586f00 x1623662818461008/t0(0) o400->fir-MDT0001-lwp-OST0023@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478112 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1700450.142643] Lustre: 61954:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549477355/real 1549477355] req@ffff927fee580600 x1623662818460960/t0(0) o400->fir-MDT0003-lwp-OST001d@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478112 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1700450.142647] Lustre: 61954:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1700450.142654] Lustre: fir-MDT0003-lwp-OST001d: Connection to fir-MDT0003 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1700451.142664] Lustre: 61953:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549477355/real 1549477355] req@ffff927fee583000 x1623662818460944/t0(0) o400->fir-MDT0001-lwp-OST001d@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478113 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [1700451.171245] Lustre: fir-MDT0001-lwp-OST001d: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1700451.187406] Lustre: Skipped 1 previous similar message [1700574.657504] Lustre: fir-OST001f: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1700574.666636] Lustre: Skipped 11 previous similar messages [1700574.672331] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:477816 to 0x480000400:477857 [1700576.977362] Lustre: fir-MDT0002-lwp-OST001d: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1700576.993538] Lustre: Skipped 5 previous similar messages [1700580.137308] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:135971 to 0xd40000401:136001 [1700587.922209] Lustre: fir-OST0023: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1700587.931351] Lustre: Skipped 1 previous similar message [1700588.829595] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:135971 to 0xd40000401:136001 [1700590.683117] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:477722 to 0xd40000402:477761 [1700601.376379] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:477722 to 0xd40000402:477793 [1700653.657348] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1700654.657371] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 48 seconds [1700654.667716] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [1700658.657461] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1700658.667718] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 4 previous similar messages [1700677.657890] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700682.658012] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700682.668269] LNet: 61886:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 9 previous similar messages [1700697.467652] Lustre: Failing over fir-OST001f [1700697.467653] Lustre: Failing over fir-OST0023 [1700697.467656] Lustre: Skipped 1 previous similar message [1700697.481862] Lustre: Skipped 2 previous similar messages [1700697.515639] LustreError: 34951:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST001d_UUID: namespace resource [0xcc0000402:0x6aeae:0x0].0x0 (ffff926ce18ab200) refcount nonzero (4) after lock cleanup; forcing cleanup. [1700697.536308] LustreError: 34951:0:(ldlm_resource.c:1146:ldlm_resource_complain()) Skipped 2 previous similar messages [1700697.574392] Lustre: fir-OST001d: Not available for connect from 10.9.101.28@o2ib4 (stopping) [1700697.583012] Lustre: Skipped 2 previous similar messages [1700697.806397] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.9.105.13@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700698.128249] Lustre: fir-OST0021: Not available for connect from 10.8.20.31@o2ib6 (stopping) [1700698.136776] Lustre: Skipped 28 previous similar messages [1700698.294685] Lustre: server umount fir-OST0019 complete [1700698.347887] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.8.11.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700698.365281] LustreError: Skipped 3 previous similar messages [1700698.985654] md1: detected capacity change from 64011422924800 to 0 [1700698.992032] md: md1 stopped. [1700699.136906] Lustre: fir-OST001f: Not available for connect from 10.9.107.46@o2ib4 (stopping) [1700699.145521] Lustre: Skipped 34 previous similar messages [1700699.591772] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.9.101.4@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700699.609226] LustreError: Skipped 5 previous similar messages [1700700.285679] Lustre: server umount fir-OST001b complete [1700700.934407] md11: detected capacity change from 64011422924800 to 0 [1700700.940870] md: md11 stopped. [1700701.120181] md7: detected capacity change from 64011422924800 to 0 [1700701.126543] md: md7 stopped. [1700702.089733] md3: detected capacity change from 64011422924800 to 0 [1700702.096101] md: md3 stopped. [1700702.540746] md5: detected capacity change from 64011422924800 to 0 [1700702.547112] md: md5 stopped. [1700702.559100] md9: detected capacity change from 64011422924800 to 0 [1700702.565492] md: md9 stopped. [1700705.956541] LNet: Removed LNI 10.0.10.106@o2ib7 [1700727.822802] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1700727.830696] alg: No test for adler32 (adler32-zlib) [1700728.646261] Lustre: Lustre: Build Version: 2.12.0 [1700728.790660] LNet: Using FastReg for registration [1700728.807614] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [1700732.952342] md: md7 stopped. [1700733.001291] md/raid:md7: device dm-72 operational as raid disk 0 [1700733.007479] md/raid:md7: device dm-107 operational as raid disk 9 [1700733.013748] md/raid:md7: device dm-106 operational as raid disk 8 [1700733.020022] md/raid:md7: device dm-93 operational as raid disk 7 [1700733.026225] md/raid:md7: device dm-92 operational as raid disk 6 [1700733.032405] md/raid:md7: device dm-80 operational as raid disk 5 [1700733.038586] md/raid:md7: device dm-79 operational as raid disk 4 [1700733.044786] md/raid:md7: device dm-67 operational as raid disk 3 [1700733.050970] md/raid:md7: device dm-66 operational as raid disk 2 [1700733.057150] md/raid:md7: device dm-83 operational as raid disk 1 [1700733.064173] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700733.082760] md7: detected capacity change from 0 to 64011422924800 [1700733.092567] md: md3 stopped. [1700733.107983] md/raid:md3: device dm-56 operational as raid disk 0 [1700733.114171] md/raid:md3: device dm-51 operational as raid disk 9 [1700733.120368] md/raid:md3: device dm-50 operational as raid disk 8 [1700733.126555] md/raid:md3: device dm-38 operational as raid disk 7 [1700733.132767] md/raid:md3: device dm-37 operational as raid disk 6 [1700733.138951] md/raid:md3: device dm-25 operational as raid disk 5 [1700733.145137] md/raid:md3: device dm-24 operational as raid disk 4 [1700733.151341] md/raid:md3: device dm-11 operational as raid disk 3 [1700733.157522] md/raid:md3: device dm-10 operational as raid disk 2 [1700733.163710] md/raid:md3: device dm-57 operational as raid disk 1 [1700733.170634] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700733.189144] md3: detected capacity change from 0 to 64011422924800 [1700733.201754] md: md1 stopped. [1700733.269052] md/raid:md1: device dm-12 operational as raid disk 0 [1700733.275242] md/raid:md1: device dm-47 operational as raid disk 9 [1700733.281434] md/raid:md1: device dm-46 operational as raid disk 8 [1700733.287633] md/raid:md1: device dm-33 operational as raid disk 7 [1700733.293828] md/raid:md1: device dm-32 operational as raid disk 6 [1700733.300041] md/raid:md1: device dm-20 operational as raid disk 5 [1700733.306248] md/raid:md1: device dm-19 operational as raid disk 4 [1700733.312446] md/raid:md1: device dm-7 operational as raid disk 3 [1700733.318553] md/raid:md1: device dm-6 operational as raid disk 2 [1700733.324674] md/raid:md1: device dm-23 operational as raid disk 1 [1700733.331981] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700733.351442] md1: detected capacity change from 0 to 64011422924800 [1700733.361973] md: md5 stopped. [1700733.381249] md/raid:md5: device dm-2 operational as raid disk 0 [1700733.387352] md/raid:md5: device dm-55 operational as raid disk 9 [1700733.393621] md/raid:md5: device dm-54 operational as raid disk 8 [1700733.399805] md/raid:md5: device dm-42 operational as raid disk 7 [1700733.406033] md/raid:md5: device dm-41 operational as raid disk 6 [1700733.412221] md/raid:md5: device dm-29 operational as raid disk 5 [1700733.418433] md/raid:md5: device dm-28 operational as raid disk 4 [1700733.424618] md/raid:md5: device dm-16 operational as raid disk 3 [1700733.430813] md/raid:md5: device dm-15 operational as raid disk 2 [1700733.437027] md/raid:md5: device dm-3 operational as raid disk 1 [1700733.443955] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700733.462918] md5: detected capacity change from 0 to 64011422924800 [1700733.484490] md: md11 stopped. [1700733.515619] md/raid:md11: device dm-62 operational as raid disk 0 [1700733.521897] md/raid:md11: device dm-115 operational as raid disk 9 [1700733.528271] md/raid:md11: device dm-114 operational as raid disk 8 [1700733.534627] md/raid:md11: device dm-102 operational as raid disk 7 [1700733.540997] md/raid:md11: device dm-101 operational as raid disk 6 [1700733.547366] md/raid:md11: device dm-89 operational as raid disk 5 [1700733.553647] md/raid:md11: device dm-88 operational as raid disk 4 [1700733.559924] md/raid:md11: device dm-76 operational as raid disk 3 [1700733.566193] md/raid:md11: device dm-75 operational as raid disk 2 [1700733.572476] md/raid:md11: device dm-63 operational as raid disk 1 [1700733.579543] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700733.602174] md11: detected capacity change from 0 to 64011422924800 [1700733.615651] md: md9 stopped. [1700733.695428] md/raid:md9: device dm-116 operational as raid disk 0 [1700733.701708] md/raid:md9: device dm-111 operational as raid disk 9 [1700733.707989] md/raid:md9: device dm-110 operational as raid disk 8 [1700733.714276] md/raid:md9: device dm-98 operational as raid disk 7 [1700733.720484] md/raid:md9: device dm-97 operational as raid disk 6 [1700733.726683] md/raid:md9: device dm-85 operational as raid disk 5 [1700733.732908] md/raid:md9: device dm-84 operational as raid disk 4 [1700733.739117] md/raid:md9: device dm-71 operational as raid disk 3 [1700733.745312] md/raid:md9: device dm-70 operational as raid disk 2 [1700733.751541] md/raid:md9: device dm-117 operational as raid disk 1 [1700733.771933] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1700733.824383] md9: detected capacity change from 0 to 64011422924800 [1700733.986371] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [1700734.335351] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700734.367354] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [1700734.414386] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1700734.747016] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700734.748204] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700734.772382] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [1700735.103903] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700735.175428] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [1700735.536974] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700735.592380] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1700735.913734] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1700741.790379] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1700742.091055] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.107.42@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700742.108607] LustreError: Skipped 1 previous similar message [1700742.642760] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.12.8@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700742.660123] LustreError: Skipped 47 previous similar messages [1700743.805126] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.7.3@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700743.822411] LustreError: Skipped 68 previous similar messages [1700745.877781] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.8.4.16@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700745.895149] LustreError: Skipped 168 previous similar messages [1700748.790537] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700749.790565] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1700750.058433] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.9.101.19@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700750.075979] LustreError: Skipped 354 previous similar messages [1700756.790721] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1700757.793684] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [1700757.805522] Lustre: fir-OST001f: in recovery but waiting for the first client to connect [1700758.006401] Lustre: fir-OST001f: Will be in recovery for at least 5:00, or until 423 clients reconnect [1700758.015927] Lustre: fir-OST001f: Connection restored to (at 10.9.101.32@o2ib4) [1700758.064319] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.9.101.4@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700758.081805] LustreError: Skipped 795 previous similar messages [1700758.524015] Lustre: fir-OST001f: Connection restored to (at 10.8.22.13@o2ib6) [1700758.531425] Lustre: Skipped 3 previous similar messages [1700759.606900] Lustre: fir-OST001f: Connection restored to (at 10.9.106.23@o2ib4) [1700759.614392] Lustre: Skipped 14 previous similar messages [1700760.790817] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 5 seconds [1700760.801073] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1700762.522181] Lustre: fir-OST001f: Connection restored to (at 10.8.22.20@o2ib6) [1700762.529579] Lustre: Skipped 2 previous similar messages [1700766.720992] Lustre: fir-OST001f: Connection restored to (at 10.9.103.25@o2ib4) [1700766.728486] Lustre: Skipped 1 previous similar message [1700767.793108] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [1700767.802835] Lustre: fir-OST0019: in recovery but waiting for the first client to connect [1700767.898554] Lustre: fir-OST0019: Will be in recovery for at least 5:00, or until 437 clients reconnect [1700768.163502] Lustre: fir-OST001b: Not available for connect from 10.8.7.24@o2ib6 (not set up) [1700773.791119] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1700773.801373] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1700774.068160] LustreError: 137-5: fir-OST001d_UUID: not available for connect from 10.9.102.43@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1700774.085705] LustreError: Skipped 427 previous similar messages [1700774.801235] Lustre: fir-OST0019: Connection restored to (at 10.8.10.32@o2ib6) [1700774.808644] Lustre: Skipped 205 previous similar messages [1700777.793396] Lustre: fir-OST001b: Imperative Recovery not enabled, recovery window 300-900 [1700777.805954] Lustre: fir-OST001b: in recovery but waiting for the first client to connect [1700777.867026] Lustre: fir-OST001b: Will be in recovery for at least 5:00, or until 386 clients reconnect [1700778.166242] Lustre: fir-OST001d: Not available for connect from 10.8.12.20@o2ib6 (not set up) [1700782.793477] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [1700782.804037] Lustre: fir-OST001d: in recovery but waiting for the first client to connect [1700782.825959] Lustre: fir-OST001d: Will be in recovery for at least 5:00, or until 408 clients reconnect [1700783.042727] Lustre: fir-OST0023: Not available for connect from 10.8.1.5@o2ib6 (not set up) [1700783.051253] Lustre: Skipped 2 previous similar messages [1700787.793586] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [1700787.802600] Lustre: fir-OST0023: in recovery but waiting for the first client to connect [1700789.682567] Lustre: fir-OST0023: Will be in recovery for at least 5:00, or until 442 clients reconnect [1700789.791487] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 16 seconds [1700789.801828] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 11 previous similar messages [1700791.536223] Lustre: fir-OST0019: Connection restored to (at 10.9.102.60@o2ib4) [1700791.543708] Lustre: Skipped 826 previous similar messages [1700792.168250] Lustre: fir-OST001f: Recovery over after 0:34, of 423 clients 423 recovered and 0 were evicted. [1700793.029267] Lustre: fir-OST0019: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 437 known clients (238 recovered, 173 in progress, and 0 evicted) already passed deadline 0:24 [1700793.050019] Lustre: Skipped 2 previous similar messages [1700795.318737] Lustre: fir-OST0019: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 437 known clients (239 recovered, 173 in progress, and 0 evicted) already passed deadline 0:27 [1700795.338355] Lustre: Skipped 2 previous similar messages [1700796.637075] Lustre: fir-OST0019: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 437 known clients (239 recovered, 173 in progress, and 0 evicted) already passed deadline 0:28 [1700796.656695] Lustre: Skipped 3 previous similar messages [1700801.600814] Lustre: fir-OST0019: Denying connection for new client 1e03043d-571f-f709-9076-72fb7d056ac3(at 10.9.103.43@o2ib4), waiting for 437 known clients (239 recovered, 173 in progress, and 0 evicted) already passed deadline 0:33 [1700801.621655] Lustre: Skipped 3 previous similar messages [1700808.675858] Lustre: fir-OST0019: Recovery over after 0:40, of 437 clients 437 recovered and 0 were evicted. [1700812.394466] Lustre: fir-OST001d: Denying connection for new client 422c37d0-97a3-ea50-ac82-e8f84679c682(at 10.8.26.2@o2ib6), waiting for 408 known clients (137 recovered, 91 in progress, and 0 evicted) already passed deadline 0:29 [1700812.415043] Lustre: Skipped 3 previous similar messages [1700820.397352] Lustre: fir-OST0021: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 425 known clients (63 recovered, 31 in progress, and 0 evicted) already passed deadline 0:27 [1700820.416805] Lustre: Skipped 6 previous similar messages [1700823.548605] Lustre: fir-OST0021: Connection restored to (at 10.8.22.14@o2ib6) [1700823.556007] Lustre: Skipped 856 previous similar messages [1700833.341491] Lustre: fir-OST0023: Recovery over after 0:44, of 442 clients 442 recovered and 0 were evicted. [1700837.483110] Lustre: fir-OST001d: Denying connection for new client 422c37d0-97a3-ea50-ac82-e8f84679c682(at 10.8.26.2@o2ib6), waiting for 408 known clients (251 recovered, 156 in progress, and 0 evicted) already passed deadline 0:54 [1700837.503793] Lustre: Skipped 11 previous similar messages [1700870.574772] Lustre: fir-OST0021: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 425 known clients (271 recovered, 153 in progress, and 0 evicted) already passed deadline 1:17 [1700870.594401] Lustre: Skipped 21 previous similar messages [1700888.684273] Lustre: fir-OST0023: Connection restored to (at 10.8.10.15@o2ib6) [1700888.691681] Lustre: Skipped 624 previous similar messages [1700937.837623] Lustre: fir-OST001b: Denying connection for new client 422c37d0-97a3-ea50-ac82-e8f84679c682(at 10.8.26.2@o2ib6), waiting for 386 known clients (243 recovered, 142 in progress, and 0 evicted) already passed deadline 2:39 [1700937.858283] Lustre: Skipped 62 previous similar messages [1701050.075260] Lustre: fir-OST0023: Connection restored to 9a86fbfd-6aa1-17d8-3e18-f77ffac8735a (at 10.8.21.25@o2ib6) [1701050.085781] Lustre: Skipped 41 previous similar messages [1701068.993069] Lustre: fir-OST001b: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 386 known clients (243 recovered, 142 in progress, and 0 evicted) already passed deadline 4:50 [1701069.013820] Lustre: Skipped 119 previous similar messages [1701078.165145] Lustre: fir-OST001b: recovery is timed out, evict stale exports [1701078.172370] Lustre: fir-OST001b: disconnecting 1 stale clients [1701078.181458] Lustre: fir-OST001b: Recovery already passed deadline 4:00, It is most likely due to DNE recovery is failed or stuck, please wait a few more minutes or abort the recovery. [1701078.355957] Lustre: fir-OST001b: Recovery over after 5:00, of 386 clients 385 recovered and 1 was evicted. [1701083.285253] Lustre: fir-OST001d: recovery is timed out, evict stale exports [1701083.292483] Lustre: fir-OST001d: disconnecting 1 stale clients [1701083.299066] Lustre: fir-OST001d: Recovery already passed deadline 4:00, It is most likely due to DNE recovery is failed or stuck, please wait a few more minutes or abort the recovery. [1701083.315574] Lustre: Skipped 1 previous similar message [1701083.396188] Lustre: fir-OST001d: Recovery over after 5:00, of 408 clients 407 recovered and 1 was evicted. [1701093.525492] Lustre: fir-OST0021: recovery is timed out, evict stale exports [1701093.532724] Lustre: fir-OST0021: disconnecting 1 stale clients [1701093.657835] Lustre: fir-OST0021: Recovery over after 5:00, of 425 clients 424 recovered and 1 was evicted. [1701294.803143] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 7 seconds [1701294.813398] LNet: 35692:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 4 previous similar messages [1701294.822908] Lustre: 35731:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549478956/real 1549478957] req@ffff9264556dfb00 x1624745852283152/t0(0) o400->fir-MDT0003-lwp-OST001d@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479712 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1701294.822924] Lustre: fir-MDT0001-lwp-OST001d: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1701294.867977] Lustre: 35731:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1701295.803211] Lustre: 35739:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549478956/real 1549478958] req@ffff9264556d8000 x1624745852283280/t0(0) o400->fir-MDT0003-lwp-OST0021@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479712 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1701295.803220] Lustre: fir-MDT0001-lwp-OST0021: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1701295.803222] Lustre: Skipped 5 previous similar messages [1701295.853648] Lustre: 35739:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [1701299.804287] Lustre: 35731:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549478956/real 1549478962] req@ffff9264556dec00 x1624745852282960/t0(0) o400->fir-MDT0003-lwp-OST001f@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479712 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1701299.804292] Lustre: fir-MDT0001-lwp-OST001f: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1701299.804293] Lustre: Skipped 3 previous similar messages [1701299.854721] Lustre: 35731:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1701308.811463] Lustre: 45351:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549478965/real 1549478965] req@ffff926c6d178900 x1624745852283520/t0(0) o39->fir-MDT0002-lwp-OST001d@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478971 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1701308.839926] Lustre: 45351:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1701314.812603] Lustre: 45353:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549478971/real 1549478971] req@ffff928c8c1c0300 x1624745852283584/t0(0) o39->fir-MDT0000-lwp-OST001b@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549478977 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1701314.818074] Lustre: Failing over fir-OST0023 [1701314.832601] LustreError: 45354:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST0019_UUID: namespace resource [0xa5956:0x0:0x0].0x0 (ffff928a0e576a80) refcount nonzero (2) after lock cleanup; forcing cleanup. [1701314.865471] Lustre: 45353:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1701315.591825] Lustre: server umount fir-OST001f complete [1701315.775501] Lustre: fir-OST001b: Not available for connect from 10.9.101.52@o2ib4 (stopping) [1701315.784114] Lustre: Skipped 1 previous similar message [1701315.844947] md7: detected capacity change from 64011422924800 to 0 [1701315.851319] md: md7 stopped. [1701316.131820] LustreError: 137-5: fir-OST001f_UUID: not available for connect from 10.8.23.21@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701316.149277] LustreError: Skipped 1736 previous similar messages [1701317.638717] Lustre: server umount fir-OST0021 complete [1701318.009996] md9: detected capacity change from 64011422924800 to 0 [1701318.016370] md: md9 stopped. [1701318.032953] md1: detected capacity change from 64011422924800 to 0 [1701318.039313] md: md1 stopped. [1701318.081225] md11: detected capacity change from 64011422924800 to 0 [1701318.082431] md3: detected capacity change from 64011422924800 to 0 [1701318.082440] md: md3 stopped. [1701318.097101] md: md11 stopped. [1701318.175992] md5: detected capacity change from 64011422924800 to 0 [1701318.182357] md: md5 stopped. [1701319.888290] LNetError: 38995:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.204@o2ib7 on NA (ib0:1:10.0.10.106): bad dst nid 10.0.10.106@o2ib7 [1701320.646300] LNetError: 38995:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.203@o2ib7 on NA (ib0:1:10.0.10.106): bad dst nid 10.0.10.106@o2ib7 [1701320.661955] LNetError: 38995:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Skipped 12 previous similar messages [1701321.860755] LNet: Removed LNI 10.0.10.106@o2ib7 [1701374.366799] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1701374.374654] alg: No test for adler32 (adler32-zlib) [1701375.178353] Lustre: Lustre: Build Version: 2.12.0 [1701375.308019] LNet: Using FastReg for registration [1701375.325618] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [1701390.566316] md: md5 stopped. [1701390.589732] md/raid:md5: device dm-2 operational as raid disk 0 [1701390.595829] md/raid:md5: device dm-55 operational as raid disk 9 [1701390.602022] md/raid:md5: device dm-54 operational as raid disk 8 [1701390.608203] md/raid:md5: device dm-42 operational as raid disk 7 [1701390.614392] md/raid:md5: device dm-41 operational as raid disk 6 [1701390.620580] md/raid:md5: device dm-29 operational as raid disk 5 [1701390.626766] md/raid:md5: device dm-28 operational as raid disk 4 [1701390.632946] md/raid:md5: device dm-16 operational as raid disk 3 [1701390.639125] md/raid:md5: device dm-15 operational as raid disk 2 [1701390.645305] md/raid:md5: device dm-3 operational as raid disk 1 [1701390.652310] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701390.671625] md5: detected capacity change from 0 to 64011422924800 [1701390.685693] md: md7 stopped. [1701390.719634] md/raid:md7: device dm-72 operational as raid disk 0 [1701390.725828] md/raid:md7: device dm-107 operational as raid disk 9 [1701390.732102] md/raid:md7: device dm-106 operational as raid disk 8 [1701390.738375] md/raid:md7: device dm-93 operational as raid disk 7 [1701390.744561] md/raid:md7: device dm-92 operational as raid disk 6 [1701390.750750] md/raid:md7: device dm-80 operational as raid disk 5 [1701390.756938] md/raid:md7: device dm-79 operational as raid disk 4 [1701390.763126] md/raid:md7: device dm-67 operational as raid disk 3 [1701390.769315] md/raid:md7: device dm-66 operational as raid disk 2 [1701390.775502] md/raid:md7: device dm-83 operational as raid disk 1 [1701390.782478] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701390.808178] md7: detected capacity change from 0 to 64011422924800 [1701390.828174] md: md11 stopped. [1701390.877040] md/raid:md11: device dm-62 operational as raid disk 0 [1701390.883323] md/raid:md11: device dm-115 operational as raid disk 9 [1701390.889680] md/raid:md11: device dm-114 operational as raid disk 8 [1701390.896040] md/raid:md11: device dm-102 operational as raid disk 7 [1701390.902393] md/raid:md11: device dm-101 operational as raid disk 6 [1701390.908747] md/raid:md11: device dm-89 operational as raid disk 5 [1701390.915020] md/raid:md11: device dm-88 operational as raid disk 4 [1701390.921295] md/raid:md11: device dm-76 operational as raid disk 3 [1701390.927573] md/raid:md11: device dm-75 operational as raid disk 2 [1701390.933850] md/raid:md11: device dm-63 operational as raid disk 1 [1701390.941093] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701390.960549] md11: detected capacity change from 0 to 64011422924800 [1701390.971229] md: md1 stopped. [1701391.011180] md/raid:md1: device dm-12 operational as raid disk 0 [1701391.017379] md/raid:md1: device dm-47 operational as raid disk 9 [1701391.023573] md/raid:md1: device dm-46 operational as raid disk 8 [1701391.029764] md/raid:md1: device dm-33 operational as raid disk 7 [1701391.035957] md/raid:md1: device dm-32 operational as raid disk 6 [1701391.042151] md/raid:md1: device dm-20 operational as raid disk 5 [1701391.048341] md/raid:md1: device dm-19 operational as raid disk 4 [1701391.054536] md/raid:md1: device dm-7 operational as raid disk 3 [1701391.060642] md/raid:md1: device dm-6 operational as raid disk 2 [1701391.066748] md/raid:md1: device dm-23 operational as raid disk 1 [1701391.073769] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701391.091389] md1: detected capacity change from 0 to 64011422924800 [1701391.114531] md: md9 stopped. [1701391.210037] md/raid:md9: device dm-116 operational as raid disk 0 [1701391.216331] md/raid:md9: device dm-111 operational as raid disk 9 [1701391.222624] md/raid:md9: device dm-110 operational as raid disk 8 [1701391.228903] md/raid:md9: device dm-98 operational as raid disk 7 [1701391.235114] md/raid:md9: device dm-97 operational as raid disk 6 [1701391.241333] md/raid:md9: device dm-85 operational as raid disk 5 [1701391.247548] md/raid:md9: device dm-84 operational as raid disk 4 [1701391.253763] md/raid:md9: device dm-71 operational as raid disk 3 [1701391.259977] md/raid:md9: device dm-70 operational as raid disk 2 [1701391.266175] md/raid:md9: device dm-117 operational as raid disk 1 [1701391.273233] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701391.331531] md9: detected capacity change from 0 to 64011422924800 [1701391.349514] md: md3 stopped. [1701391.414515] md/raid:md3: device dm-56 operational as raid disk 0 [1701391.420705] md/raid:md3: device dm-51 operational as raid disk 9 [1701391.426900] md/raid:md3: device dm-50 operational as raid disk 8 [1701391.433092] md/raid:md3: device dm-38 operational as raid disk 7 [1701391.439279] md/raid:md3: device dm-37 operational as raid disk 6 [1701391.445468] md/raid:md3: device dm-25 operational as raid disk 5 [1701391.451658] md/raid:md3: device dm-24 operational as raid disk 4 [1701391.457855] md/raid:md3: device dm-11 operational as raid disk 3 [1701391.464050] md/raid:md3: device dm-10 operational as raid disk 2 [1701391.470243] md/raid:md3: device dm-57 operational as raid disk 1 [1701391.477344] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1701391.495534] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [1701391.519196] md3: detected capacity change from 0 to 64011422924800 [1701391.542654] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [1701391.797536] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [1701391.817084] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701392.009572] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1701392.095883] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701392.102449] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701392.325142] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701392.441589] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1701392.475588] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [1701392.729609] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701392.806213] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1701398.307536] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1701398.335621] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.8.21.1@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701398.352993] LustreError: Skipped 2 previous similar messages [1701398.496830] Lustre: fir-OST001f: Not available for connect from 10.8.13.16@o2ib6 (not set up) [1701398.861471] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.8.2.19@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701398.878840] LustreError: Skipped 53 previous similar messages [1701399.020476] Lustre: fir-OST001f: Not available for connect from 10.8.2.22@o2ib6 (not set up) [1701399.029090] Lustre: Skipped 7 previous similar messages [1701399.895741] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.8.24.24@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701399.913192] LustreError: Skipped 54 previous similar messages [1701401.927600] LustreError: 137-5: fir-OST0023_UUID: not available for connect from 10.8.13.1@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701401.944967] LustreError: Skipped 210 previous similar messages [1701405.307702] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1701405.939757] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.8.13.20@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701405.957210] LustreError: Skipped 240 previous similar messages [1701407.307745] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1701409.307785] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 4 seconds [1701409.318043] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1701409.329587] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [1701409.338594] Lustre: fir-OST001f: in recovery but waiting for the first client to connect [1701410.116043] Lustre: fir-OST001f: Will be in recovery for at least 5:00, or until 229 clients reconnect [1701410.125596] Lustre: fir-OST001f: Connection restored to (at 10.9.104.71@o2ib4) [1701413.307880] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 8 seconds [1701413.318136] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1701413.951458] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.8.21.14@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701413.968914] LustreError: Skipped 111 previous similar messages [1701414.309965] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [1701414.318968] Lustre: fir-OST0023: in recovery but waiting for the first client to connect [1701419.310077] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [1701419.319076] Lustre: fir-OST001d: in recovery but waiting for the first client to connect [1701422.308088] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 17 seconds [1701422.318430] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [1701423.526476] Lustre: fir-OST001d: Will be in recovery for at least 5:00, or until 206 clients reconnect [1701423.526656] Lustre: fir-OST001f: Connection restored to (at 10.8.23.21@o2ib6) [1701423.543354] Lustre: Skipped 1 previous similar message [1701424.310138] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [1701424.319184] Lustre: fir-OST0019: in recovery but waiting for the first client to connect [1701424.588215] Lustre: fir-OST001f: Connection restored to (at 10.8.12.11@o2ib6) [1701424.595616] Lustre: Skipped 3 previous similar messages [1701426.594515] Lustre: fir-OST001f: Connection restored to (at 10.8.20.31@o2ib6) [1701426.601919] Lustre: Skipped 43 previous similar messages [1701429.310264] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [1701429.319268] Lustre: fir-OST0021: in recovery but waiting for the first client to connect [1701429.970758] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.19.1@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1701429.988137] LustreError: Skipped 970 previous similar messages [1701430.604871] Lustre: fir-OST001f: Connection restored to (at 10.8.21.25@o2ib6) [1701430.612283] Lustre: Skipped 46 previous similar messages [1701435.630798] Lustre: fir-OST0021: Will be in recovery for at least 5:00, or until 221 clients reconnect [1701437.715843] Lustre: fir-OST0019: Will be in recovery for at least 5:00, or until 231 clients reconnect [1701437.725332] Lustre: Skipped 1 previous similar message [1701438.753399] Lustre: fir-OST0019: Connection restored to (at 10.8.23.3@o2ib6) [1701438.760742] Lustre: Skipped 62 previous similar messages [1701449.701945] Lustre: fir-OST001f: Recovery over after 0:39, of 229 clients 229 recovered and 0 were evicted. [1701450.786399] Lustre: fir-OST0019: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 231 known clients (61 recovered, 73 in progress, and 0 evicted) already passed deadline 0:12 [1701450.805849] Lustre: Skipped 3 previous similar messages [1701451.848885] Lustre: fir-OST0019: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 231 known clients (74 recovered, 86 in progress, and 0 evicted) already passed deadline 0:13 [1701451.868361] Lustre: Skipped 3 previous similar messages [1701454.758182] Lustre: fir-OST001b: Connection restored to (at 10.9.104.50@o2ib4) [1701454.765666] Lustre: Skipped 1010 previous similar messages [1701458.961930] Lustre: fir-OST0023: Recovery over after 0:36, of 243 clients 243 recovered and 0 were evicted. [1701462.542860] Lustre: fir-OST0019: Recovery over after 0:25, of 231 clients 231 recovered and 0 were evicted. [1701462.552778] Lustre: Skipped 1 previous similar message [1701473.531292] Lustre: fir-OST001b: Recovery over after 0:36, of 187 clients 187 recovered and 0 were evicted. [1701555.074852] Lustre: fir-OST001f: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.52@o2ib7) [1701555.084258] Lustre: Skipped 147 previous similar messages [1702262.327478] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [1702262.337820] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 11 previous similar messages [1702262.347437] Lustre: 46857:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549479924/real 1549479924] req@ffff9293efad0600 x1624746530716960/t0(0) o400->fir-MDT0001-lwp-OST0019@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549480680 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1702262.347487] Lustre: fir-MDT0001-lwp-OST001d: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1702262.347489] Lustre: Skipped 1 previous similar message [1702262.397823] Lustre: 46857:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [1702264.327530] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1702264.337786] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 4 previous similar messages [1702264.347309] Lustre: 46865:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549479924/real 1549479926] req@ffff9293efad3f00 x1624746530717104/t0(0) o400->fir-MDT0003-lwp-OST001b@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549480680 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1702264.347318] Lustre: fir-MDT0001-lwp-OST001b: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1702264.347320] Lustre: Skipped 2 previous similar messages [1702264.397774] Lustre: 46865:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1702266.327625] Lustre: 46857:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549479924/real 1549479928] req@ffff9293efad1800 x1624746530716784/t0(0) o400->fir-MDT0003-lwp-OST001f@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549480680 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1702266.327633] Lustre: fir-MDT0001-lwp-OST001f: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1702266.327635] Lustre: Skipped 4 previous similar messages [1702266.378068] Lustre: 46857:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1702268.327622] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1702268.337880] LNet: 46637:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 16 previous similar messages [1702277.197825] Lustre: 60060:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549479933/real 1549479933] req@ffff926faf840300 x1624746530717344/t0(0) o39->fir-MDT0002-lwp-OST0021@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479939 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1702277.226295] Lustre: 60060:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1702283.198960] Lustre: 60057:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1549479939/real 1549479939] req@ffff926473698000 x1624746530717424/t0(0) o39->fir-MDT0000-lwp-OST0023@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549479945 ref 2 fl Rpc:XN/0/ffffffff rc 0/-1 [1702283.204436] Lustre: Failing over fir-OST001f [1702283.231925] Lustre: 60057:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1702283.245412] LustreError: 60061:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST001b_UUID: namespace resource [0xa5944:0x0:0x0].0x0 (ffff92872423b140) refcount nonzero (2) after lock cleanup; forcing cleanup. [1702283.265380] LustreError: 60061:0:(ldlm_resource.c:1146:ldlm_resource_complain()) Skipped 118 previous similar messages [1702283.292495] Lustre: fir-OST001b: Not available for connect from 10.8.9.10@o2ib6 (stopping) [1702283.300941] Lustre: Skipped 5 previous similar messages [1702283.792041] Lustre: fir-OST0021: Not available for connect from 10.8.20.32@o2ib6 (stopping) [1702283.800570] Lustre: Skipped 19 previous similar messages [1702285.709091] Lustre: fir-OST001d: Not available for connect from 10.8.6.13@o2ib6 (stopping) [1702285.717535] Lustre: Skipped 31 previous similar messages [1702285.725031] LustreError: 137-5: fir-OST0023_UUID: not available for connect from 10.8.6.13@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702285.742412] LustreError: Skipped 136 previous similar messages [1702286.070549] Lustre: server umount fir-OST0023 complete [1702286.075868] Lustre: Skipped 3 previous similar messages [1702286.476782] md3: detected capacity change from 64011422924800 to 0 [1702286.481611] md1: detected capacity change from 64011422924800 to 0 [1702286.481621] md: md1 stopped. [1702286.483353] md7: detected capacity change from 64011422924800 to 0 [1702286.483356] md: md7 stopped. [1702286.501982] md: md3 stopped. [1702286.707621] md5: detected capacity change from 64011422924800 to 0 [1702286.710138] md9: detected capacity change from 64011422924800 to 0 [1702286.710147] md: md9 stopped. [1702286.714196] md11: detected capacity change from 64011422924800 to 0 [1702286.714198] md: md11 stopped. [1702286.732991] md: md5 stopped. [1702288.647708] LNetError: 38995:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.210@o2ib7 on NA (ib0:1:10.0.10.106): bad dst nid 10.0.10.106@o2ib7 [1702290.358127] LNet: Removed LNI 10.0.10.106@o2ib7 [1702340.605194] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1702340.613024] alg: No test for adler32 (adler32-zlib) [1702341.415187] Lustre: Lustre: Build Version: 2.12.0 [1702341.540752] LNet: Using FastReg for registration [1702341.558147] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [1702345.773568] md: md11 stopped. [1702345.817285] md/raid:md11: device dm-62 operational as raid disk 0 [1702345.823562] md/raid:md11: device dm-115 operational as raid disk 9 [1702345.829919] md/raid:md11: device dm-114 operational as raid disk 8 [1702345.836270] md/raid:md11: device dm-102 operational as raid disk 7 [1702345.842630] md/raid:md11: device dm-101 operational as raid disk 6 [1702345.848987] md/raid:md11: device dm-89 operational as raid disk 5 [1702345.855253] md/raid:md11: device dm-88 operational as raid disk 4 [1702345.861526] md/raid:md11: device dm-76 operational as raid disk 3 [1702345.867792] md/raid:md11: device dm-75 operational as raid disk 2 [1702345.874058] md/raid:md11: device dm-63 operational as raid disk 1 [1702345.881179] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702345.914617] md11: detected capacity change from 0 to 64011422924800 [1702345.931144] md: md5 stopped. [1702345.957752] md/raid:md5: device dm-2 operational as raid disk 0 [1702345.963851] md/raid:md5: device dm-55 operational as raid disk 9 [1702345.970046] md/raid:md5: device dm-54 operational as raid disk 8 [1702345.976236] md/raid:md5: device dm-42 operational as raid disk 7 [1702345.982426] md/raid:md5: device dm-41 operational as raid disk 6 [1702345.988615] md/raid:md5: device dm-29 operational as raid disk 5 [1702345.994847] md/raid:md5: device dm-28 operational as raid disk 4 [1702346.001032] md/raid:md5: device dm-16 operational as raid disk 3 [1702346.007281] md/raid:md5: device dm-15 operational as raid disk 2 [1702346.013471] md/raid:md5: device dm-3 operational as raid disk 1 [1702346.020632] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702346.049619] md5: detected capacity change from 0 to 64011422924800 [1702346.065685] md: md1 stopped. [1702346.121225] md/raid:md1: device dm-12 operational as raid disk 0 [1702346.127423] md/raid:md1: device dm-47 operational as raid disk 9 [1702346.133619] md/raid:md1: device dm-46 operational as raid disk 8 [1702346.139814] md/raid:md1: device dm-33 operational as raid disk 7 [1702346.146010] md/raid:md1: device dm-32 operational as raid disk 6 [1702346.152208] md/raid:md1: device dm-20 operational as raid disk 5 [1702346.158403] md/raid:md1: device dm-19 operational as raid disk 4 [1702346.164598] md/raid:md1: device dm-7 operational as raid disk 3 [1702346.170710] md/raid:md1: device dm-6 operational as raid disk 2 [1702346.176811] md/raid:md1: device dm-23 operational as raid disk 1 [1702346.184156] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702346.216994] md1: detected capacity change from 0 to 64011422924800 [1702346.231785] md: md3 stopped. [1702346.274704] md/raid:md3: device dm-56 operational as raid disk 0 [1702346.280902] md/raid:md3: device dm-51 operational as raid disk 9 [1702346.287133] md/raid:md3: device dm-50 operational as raid disk 8 [1702346.293341] md/raid:md3: device dm-38 operational as raid disk 7 [1702346.299544] md/raid:md3: device dm-37 operational as raid disk 6 [1702346.305746] md/raid:md3: device dm-25 operational as raid disk 5 [1702346.311941] md/raid:md3: device dm-24 operational as raid disk 4 [1702346.318138] md/raid:md3: device dm-11 operational as raid disk 3 [1702346.324332] md/raid:md3: device dm-10 operational as raid disk 2 [1702346.330527] md/raid:md3: device dm-57 operational as raid disk 1 [1702346.338000] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702346.373794] md3: detected capacity change from 0 to 64011422924800 [1702346.394905] md: md9 stopped. [1702346.465522] md/raid:md9: device dm-116 operational as raid disk 0 [1702346.471795] md/raid:md9: device dm-111 operational as raid disk 9 [1702346.478078] md/raid:md9: device dm-110 operational as raid disk 8 [1702346.484357] md/raid:md9: device dm-98 operational as raid disk 7 [1702346.490542] md/raid:md9: device dm-97 operational as raid disk 6 [1702346.496735] md/raid:md9: device dm-85 operational as raid disk 5 [1702346.502929] md/raid:md9: device dm-84 operational as raid disk 4 [1702346.509120] md/raid:md9: device dm-71 operational as raid disk 3 [1702346.515318] md/raid:md9: device dm-70 operational as raid disk 2 [1702346.521511] md/raid:md9: device dm-117 operational as raid disk 1 [1702346.528728] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702346.564136] md9: detected capacity change from 0 to 64011422924800 [1702346.575877] md: md7 stopped. [1702346.620839] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [1702346.631518] md/raid:md7: device dm-72 operational as raid disk 0 [1702346.637706] md/raid:md7: device dm-107 operational as raid disk 9 [1702346.643984] md/raid:md7: device dm-106 operational as raid disk 8 [1702346.650258] md/raid:md7: device dm-93 operational as raid disk 7 [1702346.656448] md/raid:md7: device dm-92 operational as raid disk 6 [1702346.662637] md/raid:md7: device dm-80 operational as raid disk 5 [1702346.668833] md/raid:md7: device dm-79 operational as raid disk 4 [1702346.675025] md/raid:md7: device dm-67 operational as raid disk 3 [1702346.681214] md/raid:md7: device dm-66 operational as raid disk 2 [1702346.687436] md/raid:md7: device dm-83 operational as raid disk 1 [1702346.694365] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1702346.771563] md7: detected capacity change from 0 to 64011422924800 [1702346.829598] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [1702346.962198] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702347.028648] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1702347.067605] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [1702347.177213] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702347.332641] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702347.353730] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1702347.407756] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702347.653717] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702347.684680] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [1702347.968867] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1702349.540498] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1702350.562691] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.8.27.19@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702350.562805] Lustre: fir-OST0023: Not available for connect from 10.8.27.19@o2ib6 (not set up) [1702350.588844] LustreError: Skipped 1 previous similar message [1702352.400304] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.9.104.71@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702352.417870] LustreError: Skipped 7 previous similar messages [1702354.091505] LustreError: 137-5: fir-OST001d_UUID: not available for connect from 10.9.107.6@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702354.108960] LustreError: Skipped 2 previous similar messages [1702356.189296] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.8.12.8@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702356.206668] LustreError: Skipped 25 previous similar messages [1702356.540663] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1702360.309775] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.8.11.18@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702360.327234] LustreError: Skipped 55 previous similar messages [1702362.540804] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1702365.540867] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [1702365.551121] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1702365.562635] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [1702365.571638] Lustre: fir-OST0023: in recovery but waiting for the first client to connect [1702365.574945] Lustre: fir-OST0023: Will be in recovery for at least 5:00, or until 125 clients reconnect [1702365.575081] Lustre: fir-OST0023: Connection restored to 029e2f9e-d883-e8d0-d1e8-9a1826095341 (at 10.8.30.33@o2ib6) [1702366.148155] Lustre: fir-OST0023: Connection restored to (at 10.8.10.28@o2ib6) [1702366.155563] Lustre: Skipped 9 previous similar messages [1702367.259575] Lustre: fir-OST0023: Connection restored to b30edc85-7a93-af90-f7e4-3b5a24d9d571 (at 10.8.23.15@o2ib6) [1702367.270102] Lustre: Skipped 11 previous similar messages [1702368.334206] LustreError: 137-5: fir-OST0021_UUID: not available for connect from 10.8.10.7@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702368.351570] LustreError: Skipped 454 previous similar messages [1702369.376317] Lustre: fir-OST0023: Connection restored to (at 10.8.20.31@o2ib6) [1702369.383715] Lustre: Skipped 17 previous similar messages [1702370.540981] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 8 seconds [1702370.551238] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1702370.562812] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [1702370.571802] Lustre: fir-OST001d: in recovery but waiting for the first client to connect [1702370.600289] Lustre: fir-OST001d: Will be in recovery for at least 5:00, or until 109 clients reconnect [1702373.454394] Lustre: fir-OST0023: Connection restored to (at 10.8.10.13@o2ib6) [1702373.461802] Lustre: Skipped 43 previous similar messages [1702375.543159] Lustre: fir-OST0019: Imperative Recovery not enabled, recovery window 300-900 [1702375.552141] Lustre: fir-OST0019: in recovery but waiting for the first client to connect [1702379.541187] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 17 seconds [1702379.551534] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 6 previous similar messages [1702380.543235] Lustre: fir-OST001b: Imperative Recovery not enabled, recovery window 300-900 [1702380.552233] Lustre: fir-OST001b: in recovery but waiting for the first client to connect [1702381.820527] Lustre: fir-OST0023: Connection restored to (at 10.8.1.24@o2ib6) [1702381.827844] Lustre: Skipped 26 previous similar messages [1702384.378073] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.8.11.33@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1702384.395556] LustreError: Skipped 75 previous similar messages [1702390.686836] Lustre: fir-OST001b: Will be in recovery for at least 5:00, or until 103 clients reconnect [1702395.543720] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [1702395.555715] Lustre: fir-OST0021: in recovery but waiting for the first client to connect [1702397.541607] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1702397.551861] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 5 previous similar messages [1702400.676734] Lustre: fir-OST0021: Will be in recovery for at least 5:00, or until 113 clients reconnect [1702400.676768] Lustre: fir-OST001b: Connection restored to (at 10.8.27.19@o2ib6) [1702400.676770] Lustre: Skipped 76 previous similar messages [1702400.699116] Lustre: Skipped 1 previous similar message [1702400.928854] Lustre: fir-OST0023: Recovery over after 0:36, of 125 clients 125 recovered and 0 were evicted. [1702405.459443] Lustre: fir-OST001f: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 121 known clients (1 recovered, 4 in progress, and 0 evicted) already passed deadline 0:04 [1702405.478726] Lustre: Skipped 4 previous similar messages [1702406.095280] Lustre: fir-OST001f: Denying connection for new client fir-MDT0003-mdtlov_UUID(at 10.0.10.52@o2ib7), waiting for 121 known clients (2 recovered, 4 in progress, and 0 evicted) already passed deadline 0:04 [1702406.114552] Lustre: Skipped 3 previous similar messages [1702415.363323] Lustre: fir-OST001d: Recovery over after 0:45, of 109 clients 109 recovered and 0 were evicted. [1702418.655400] Lustre: fir-OST0021: Recovery over after 0:18, of 113 clients 113 recovered and 0 were evicted. [1702430.538759] Lustre: fir-OST001f: Denying connection for new client fir-MDT0002-mdtlov_UUID(at 10.0.10.51@o2ib7), waiting for 121 known clients (8 recovered, 102 in progress, and 0 evicted) already passed deadline 0:29 [1702441.213935] Lustre: fir-OST001f: Connection restored to (at 10.8.11.5@o2ib6) [1702441.221251] Lustre: Skipped 497 previous similar messages [1702443.381050] Lustre: fir-OST001f: Recovery over after 0:43, of 121 clients 121 recovered and 0 were evicted. [1702443.390968] Lustre: Skipped 2 previous similar messages [1702526.097499] Lustre: fir-OST001f: Connection restored to (at 10.0.10.52@o2ib7) [1702526.104896] Lustre: Skipped 12 previous similar messages [1703402.672863] Lustre: fir-OST0023: Connection restored to 5d5c5456-e139-d787-f9bb-4817666d86b7 (at 10.8.12.34@o2ib6) [1703402.683417] Lustre: Skipped 31 previous similar messages [1703402.724706] Lustre: fir-OST0019: deleting orphan objects from 0x0:678335 to 0x0:678369 [1703402.725131] Lustre: fir-OST001f: deleting orphan objects from 0x0:678331 to 0x0:678369 [1703402.736260] Lustre: fir-OST001d: deleting orphan objects from 0x0:678330 to 0x0:678369 [1703402.736455] Lustre: fir-OST001b: deleting orphan objects from 0x0:678327 to 0x0:678369 [1703402.739499] Lustre: fir-OST0021: deleting orphan objects from 0x0:678304 to 0x0:678337 [1703402.742545] Lustre: fir-OST0023: deleting orphan objects from 0x0:678387 to 0x0:678465 [1703402.794263] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:477817 to 0x640000400:477857 [1703402.794304] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:477911 to 0xd00000402:477953 [1703402.794313] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:477752 to 0xcc0000402:477825 [1703402.794319] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:477722 to 0xd40000402:477825 [1703402.794366] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:477816 to 0x480000400:477889 [1703402.809662] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:477705 to 0x980000402:477729 [1703404.006135] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:147624 to 0xcc0000400:147649 [1703404.014957] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:147623 to 0x980000400:147649 [1703404.015628] Lustre: fir-OST0023: deleting orphan objects from 0xd40000400:147561 to 0xd40000400:147585 [1703404.022155] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:147662 to 0x480000401:147681 [1703404.026217] Lustre: fir-OST001b: deleting orphan objects from 0xd00000400:147529 to 0xd00000400:147553 [1703404.035898] Lustre: fir-OST0019: deleting orphan objects from 0x640000401:147559 to 0x640000401:147585 [1703404.044791] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:135973 to 0xcc0000401:136001 [1703404.050223] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:135971 to 0xd40000401:136033 [1703404.051782] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:135977 to 0x980000401:136001 [1703404.063427] Lustre: fir-OST0019: deleting orphan objects from 0x640000402:135814 to 0x640000402:135841 [1703404.078745] Lustre: fir-OST001b: deleting orphan objects from 0xd00000401:135813 to 0xd00000401:135841 [1703404.095942] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:135969 to 0x480000402:136001 [1703418.766323] Lustre: fir-OST0019: Connection restored to 56a7901d-9f1f-93d3-c75e-92d9b6eaca50 (at 10.9.107.48@o2ib4) [1703418.777068] Lustre: Skipped 2086 previous similar messages [1703452.770577] Lustre: fir-OST0023: Connection restored to (at 10.9.104.51@o2ib4) [1703452.778062] Lustre: Skipped 312 previous similar messages [1703518.273373] Lustre: fir-OST001f: Connection restored to 838e95ba-7d72-299c-b143-b6a5f04f9e78 (at 10.9.107.36@o2ib4) [1703518.284016] Lustre: Skipped 128 previous similar messages [1703646.299336] Lustre: fir-OST0019: Connection restored to 3d5ac7cc-5cb1-6aef-d2d6-2af538ce79c6 (at 10.8.25.22@o2ib6) [1703646.309896] Lustre: Skipped 409 previous similar messages [1703763.917722] Lustre: fir-OST0023: haven't heard from client d9d906b5-22c6-448d-c0e5-c1df58bf3870 (at 10.8.31.4@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92647a6a2800, cur 1549481426 expire 1549481276 last 1549481199 [1703781.939563] Lustre: fir-OST001b: haven't heard from client d9d906b5-22c6-448d-c0e5-c1df58bf3870 (at 10.8.31.4@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92947ae91c00, cur 1549481444 expire 1549481294 last 1549481217 [1703903.659773] Lustre: fir-OST001f: Connection restored to (at 10.9.107.8@o2ib4) [1703903.667181] Lustre: Skipped 762 previous similar messages [1704415.700735] Lustre: fir-OST001f: Connection restored to c2c0536b-b17c-9118-170c-d010e3c3b183 (at 10.9.107.15@o2ib4) [1704415.711452] Lustre: Skipped 1117 previous similar messages [1704742.868384] Lustre: fir-OST0023: haven't heard from client f6aefb2b-cc93-9031-e9bd-7f3143d0a97d (at 10.8.11.9@o2ib6) in 227 seconds. I think it's dead, and I am evicting it. exp ffff92847a5fcc00, cur 1549482405 expire 1549482255 last 1549482178 [1705015.955316] Lustre: fir-OST001d: Connection restored to 73232f55-96e8-5448-4cde-7ce2caf43268 (at 10.8.7.8@o2ib6) [1705015.965665] Lustre: Skipped 1136 previous similar messages [1705616.708079] Lustre: fir-OST001d: Connection restored to (at 10.8.18.19@o2ib6) [1705616.715484] Lustre: Skipped 937 previous similar messages [1706216.909953] Lustre: fir-OST001d: Connection restored to 4cdb4c1f-c631-73f5-0cc6-576f1959d1eb (at 10.8.17.8@o2ib6) [1706216.920384] Lustre: Skipped 1030 previous similar messages [1706301.723409] Lustre: fir-OST0021: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706303.638155] Lustre: fir-OST001b: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706303.638278] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:479077 to 0x980000402:479201 [1706304.538363] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:479305 to 0xd00000402:479489 [1706305.437847] Lustre: fir-OST0019: deleting orphan objects from 0x0:681522 to 0x0:681665 [1706312.758577] Lustre: fir-OST001f: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706312.767717] Lustre: Skipped 1 previous similar message [1706313.396185] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:479232 to 0x480000400:479297 [1706319.632165] LNetError: 61366:0:(o2iblnd_cb.c:3324:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [1706319.642339] LNetError: 61366:0:(o2iblnd_cb.c:3399:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (3): c: 0, oc: 3, rc: 8 [1706319.654917] Lustre: 61444:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549483976/real 1549483981] req@ffff92887fe39b00 x1624747548449072/t0(0) o400->fir-MDT0002-lwp-OST0021@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484732 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706319.654953] Lustre: fir-MDT0002-lwp-OST001b: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706319.700038] Lustre: 61444:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [1706320.632189] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [1706320.642448] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 4 previous similar messages [1706320.651995] Lustre: 61436:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549483976/real 1549483982] req@ffff92887fe39e00 x1624747548449136/t0(0) o400->fir-MDT0002-lwp-OST001f@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484732 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706320.651999] Lustre: fir-MDT0000-lwp-OST001f: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706320.652001] Lustre: Skipped 7 previous similar messages [1706320.702443] Lustre: 61436:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [1706351.632900] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [1706351.643157] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 16 previous similar messages [1706351.652759] Lustre: 61441:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549484007/real 1549484013] req@ffff928a8eddf200 x1624747548745072/t0(0) o400->fir-MDT0002-lwp-OST0023@10.0.10.51@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484763 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706351.652767] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [1706351.695683] Lustre: 61441:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1706351.705523] Lustre: fir-MDT0002-lwp-OST0023: Connection to fir-MDT0002 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706351.721719] Lustre: Skipped 2 previous similar messages [1706364.048695] Lustre: fir-OST001d: Client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706364.057832] Lustre: Skipped 2 previous similar messages [1706364.633463] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706364.650946] LustreError: Skipped 292 previous similar messages [1706367.701963] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:479177 to 0xcc0000402:479329 [1706367.702043] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:479202 to 0x640000400:479265 [1706386.966987] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706386.984444] LustreError: Skipped 1 previous similar message [1706420.046577] Lustre: fir-OST0021: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706420.046605] LustreError: 137-5: fir-OST001a_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706420.073167] Lustre: Skipped 9 previous similar messages [1706420.864089] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:479305 to 0xd00000402:479521 [1706420.864127] Lustre: fir-OST0023: deleting orphan objects from 0x0:681619 to 0x0:681665 [1706421.219653] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:479077 to 0x980000402:479233 [1706421.229188] Lustre: fir-OST0021: deleting orphan objects from 0x0:681535 to 0x0:681569 [1706421.229193] Lustre: fir-OST001f: deleting orphan objects from 0x0:681529 to 0x0:681665 [1706421.592775] Lustre: fir-OST0019: deleting orphan objects from 0x0:681522 to 0x0:681697 [1706422.161898] Lustre: fir-OST001d: deleting orphan objects from 0x0:681525 to 0x0:681665 [1706422.463749] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:479169 to 0xd40000402:479233 [1706431.097112] Lustre: fir-OST0023: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706431.106243] Lustre: Skipped 2 previous similar messages [1706438.916488] Lustre: fir-OST0021: deleting orphan objects from 0x0:681535 to 0x0:681569 [1706466.623516] LustreError: 137-5: fir-OST001e_UUID: not available for connect from 10.0.10.51@o2ib7 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706466.641020] LustreError: Skipped 9 previous similar messages [1706491.711680] Lustre: fir-OST001f: Client fir-MDT0002-mdtlov_UUID (at 10.0.10.51@o2ib7) reconnecting [1706491.711950] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:479169 to 0xd40000402:479233 [1706491.712129] Lustre: fir-OST001d: deleting orphan objects from 0x0:681525 to 0x0:681697 [1706491.712202] Lustre: fir-OST001b: deleting orphan objects from 0x0:681616 to 0x0:681633 [1706491.746506] Lustre: Skipped 3 previous similar messages [1706491.752134] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:479232 to 0x480000400:479297 [1706753.048405] Lustre: fir-MDT0000-lwp-OST0023: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706753.064565] Lustre: Skipped 10 previous similar messages [1706753.642187] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 5 seconds [1706753.652440] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 3 previous similar messages [1706753.661954] Lustre: 61435:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549484415/real 1549484415] req@ffff928bd2223300 x1624747549335344/t0(0) o400->fir-MDT0001-lwp-OST0021@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484662 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706753.690872] Lustre: 61435:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1706758.642359] Lustre: 61435:0:(client.c:2132:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1549484415/real 1549484420] req@ffff928bd2225700 x1624747549335152/t0(0) o400->fir-MDT0001-lwp-OST001d@10.0.10.52@o2ib7:12/10 lens 224/224 e 0 to 1 dl 1549484662 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [1706758.642375] Lustre: fir-MDT0001-lwp-OST0023: Connection to fir-MDT0001 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [1706758.642376] Lustre: Skipped 9 previous similar messages [1706758.692831] Lustre: 61435:0:(client.c:2132:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1706786.642967] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 20 seconds [1706786.653311] LNet: 61366:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 23 previous similar messages [1706800.063563] Lustre: Failing over fir-OST001f [1706800.068020] Lustre: Skipped 4 previous similar messages [1706800.079210] Lustre: fir-OST0021: Not available for connect from 10.9.101.15@o2ib4 (stopping) [1706800.087460] LustreError: 117135:0:(ldlm_resource.c:1146:ldlm_resource_complain()) filter-fir-OST0021_UUID: namespace resource [0x980000402:0x74ee0:0x0].0x0 (ffff926afdc61bc0) refcount nonzero (2) after lock cleanup; forcing cleanup. [1706800.108581] Lustre: Skipped 3 previous similar messages [1706800.579277] Lustre: fir-OST0023: Not available for connect from 10.8.20.28@o2ib6 (stopping) [1706800.587804] Lustre: Skipped 56 previous similar messages [1706801.619930] Lustre: fir-OST0019: Not available for connect from 10.8.2.10@o2ib6 (stopping) [1706801.628371] Lustre: Skipped 97 previous similar messages [1706802.492852] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.8.7.30@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706802.510225] LustreError: Skipped 2 previous similar messages [1706802.972011] Lustre: server umount fir-OST001b complete [1706802.977331] Lustre: Skipped 1 previous similar message [1706805.488387] md7: detected capacity change from 64011422924800 to 0 [1706805.494752] md: md7 stopped. [1706805.951258] md1: detected capacity change from 64011422924800 to 0 [1706805.957628] md: md1 stopped. [1706808.184843] md9: detected capacity change from 64011422924800 to 0 [1706808.191211] md: md9 stopped. [1706810.121366] md5: detected capacity change from 64011422924800 to 0 [1706810.127759] md: md5 stopped. [1706810.198889] md11: detected capacity change from 64011422924800 to 0 [1706810.205345] md: md11 stopped. [1706810.394840] md3: detected capacity change from 64011422924800 to 0 [1706810.401206] md: md3 stopped. [1706811.878065] LNetError: 38995:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.202@o2ib7 on NA (ib0:1:10.0.10.106): bad dst nid 10.0.10.106@o2ib7 [1706812.626600] LNetError: 38995:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Can't accept conn from 10.0.10.211@o2ib7 on NA (ib0:1:10.0.10.106): bad dst nid 10.0.10.106@o2ib7 [1706812.642258] LNetError: 38995:0:(o2iblnd_cb.c:2469:kiblnd_passive_connect()) Skipped 15 previous similar messages [1706813.875570] LNet: Removed LNI 10.0.10.106@o2ib7 [1706849.328658] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [1706849.336458] alg: No test for adler32 (adler32-zlib) [1706850.140890] Lustre: Lustre: Build Version: 2.12.0 [1706850.270544] LNet: Using FastReg for registration [1706850.287713] LNet: Added LNI 10.0.10.106@o2ib7 [8/256/0/180] [1706854.556114] md: md11 stopped. [1706854.566798] md/raid:md11: device dm-62 operational as raid disk 0 [1706854.573076] md/raid:md11: device dm-115 operational as raid disk 9 [1706854.579436] md/raid:md11: device dm-114 operational as raid disk 8 [1706854.585810] md/raid:md11: device dm-102 operational as raid disk 7 [1706854.592187] md/raid:md11: device dm-101 operational as raid disk 6 [1706854.598557] md/raid:md11: device dm-89 operational as raid disk 5 [1706854.604837] md/raid:md11: device dm-88 operational as raid disk 4 [1706854.611107] md/raid:md11: device dm-76 operational as raid disk 3 [1706854.617387] md/raid:md11: device dm-75 operational as raid disk 2 [1706854.623658] md/raid:md11: device dm-63 operational as raid disk 1 [1706854.630734] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706854.649518] md11: detected capacity change from 0 to 64011422924800 [1706854.657254] md: md7 stopped. [1706854.680504] md/raid:md7: device dm-72 operational as raid disk 0 [1706854.686696] md/raid:md7: device dm-107 operational as raid disk 9 [1706854.692985] md/raid:md7: device dm-106 operational as raid disk 8 [1706854.699272] md/raid:md7: device dm-93 operational as raid disk 7 [1706854.705462] md/raid:md7: device dm-92 operational as raid disk 6 [1706854.711649] md/raid:md7: device dm-80 operational as raid disk 5 [1706854.717872] md/raid:md7: device dm-79 operational as raid disk 4 [1706854.724061] md/raid:md7: device dm-67 operational as raid disk 3 [1706854.730272] md/raid:md7: device dm-66 operational as raid disk 2 [1706854.736468] md/raid:md7: device dm-83 operational as raid disk 1 [1706854.743476] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706854.762085] md7: detected capacity change from 0 to 64011422924800 [1706854.778615] md: md9 stopped. [1706854.828666] md/raid:md9: device dm-116 operational as raid disk 0 [1706854.834943] md/raid:md9: device dm-111 operational as raid disk 9 [1706854.841220] md/raid:md9: device dm-110 operational as raid disk 8 [1706854.847498] md/raid:md9: device dm-98 operational as raid disk 7 [1706854.853688] md/raid:md9: device dm-97 operational as raid disk 6 [1706854.859873] md/raid:md9: device dm-85 operational as raid disk 5 [1706854.866053] md/raid:md9: device dm-84 operational as raid disk 4 [1706854.872233] md/raid:md9: device dm-71 operational as raid disk 3 [1706854.878413] md/raid:md9: device dm-70 operational as raid disk 2 [1706854.884600] md/raid:md9: device dm-117 operational as raid disk 1 [1706854.891713] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706854.909108] md9: detected capacity change from 0 to 64011422924800 [1706854.923122] md: md3 stopped. [1706854.964085] md/raid:md3: device dm-56 operational as raid disk 0 [1706854.970335] md/raid:md3: device dm-51 operational as raid disk 9 [1706854.976545] md/raid:md3: device dm-50 operational as raid disk 8 [1706854.982741] md/raid:md3: device dm-38 operational as raid disk 7 [1706854.988937] md/raid:md3: device dm-37 operational as raid disk 6 [1706854.995134] md/raid:md3: device dm-25 operational as raid disk 5 [1706855.001318] md/raid:md3: device dm-24 operational as raid disk 4 [1706855.007496] md/raid:md3: device dm-11 operational as raid disk 3 [1706855.013679] md/raid:md3: device dm-10 operational as raid disk 2 [1706855.019872] md/raid:md3: device dm-57 operational as raid disk 1 [1706855.028324] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706855.048778] md3: detected capacity change from 0 to 64011422924800 [1706855.063308] md: md1 stopped. [1706855.089464] md/raid:md1: device dm-12 operational as raid disk 0 [1706855.095673] md/raid:md1: device dm-47 operational as raid disk 9 [1706855.101867] md/raid:md1: device dm-46 operational as raid disk 8 [1706855.108087] md/raid:md1: device dm-33 operational as raid disk 7 [1706855.114276] md/raid:md1: device dm-32 operational as raid disk 6 [1706855.120465] md/raid:md1: device dm-20 operational as raid disk 5 [1706855.126663] md/raid:md1: device dm-19 operational as raid disk 4 [1706855.132901] md/raid:md1: device dm-7 operational as raid disk 3 [1706855.139013] md/raid:md1: device dm-6 operational as raid disk 2 [1706855.145139] md/raid:md1: device dm-23 operational as raid disk 1 [1706855.153487] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706855.170652] md1: detected capacity change from 0 to 64011422924800 [1706855.179114] md: md5 stopped. [1706855.209017] md/raid:md5: device dm-2 operational as raid disk 0 [1706855.215130] md/raid:md5: device dm-55 operational as raid disk 9 [1706855.221334] md/raid:md5: device dm-54 operational as raid disk 8 [1706855.227528] md/raid:md5: device dm-42 operational as raid disk 7 [1706855.233719] md/raid:md5: device dm-41 operational as raid disk 6 [1706855.239912] md/raid:md5: device dm-29 operational as raid disk 5 [1706855.246109] md/raid:md5: device dm-28 operational as raid disk 4 [1706855.252297] md/raid:md5: device dm-16 operational as raid disk 3 [1706855.258483] md/raid:md5: device dm-15 operational as raid disk 2 [1706855.264672] md/raid:md5: device dm-3 operational as raid disk 1 [1706855.271602] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [1706855.293462] md5: detected capacity change from 0 to 64011422924800 [1706855.608720] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [1706855.889748] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [1706855.915718] LDISKFS-fs (md9): file extents enabled, maximum tree depth=5 [1706855.955784] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706856.252157] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706856.265111] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706856.295779] LDISKFS-fs (md3): file extents enabled, maximum tree depth=5 [1706856.669481] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706856.672736] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [1706856.737769] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [1706856.979342] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706857.074334] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [1706861.269677] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1706861.296479] LustreError: 137-5: fir-OST0021_UUID: not available for connect from 10.8.17.4@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706861.747206] Lustre: fir-OST0023: Not available for connect from 10.8.7.33@o2ib6 (not set up) [1706861.869571] LustreError: 137-5: fir-OST001c_UUID: not available for connect from 10.9.101.34@o2ib4 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706861.887131] LustreError: Skipped 26 previous similar messages [1706862.417158] Lustre: fir-OST0023: Not available for connect from 10.8.23.27@o2ib6 (not set up) [1706862.425858] Lustre: Skipped 5 previous similar messages [1706862.870509] LustreError: 137-5: fir-OST001b_UUID: not available for connect from 10.8.20.31@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706862.887972] LustreError: Skipped 42 previous similar messages [1706864.875809] LustreError: 137-5: fir-OST0021_UUID: not available for connect from 10.8.24.28@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706864.893273] LustreError: Skipped 137 previous similar messages [1706868.269835] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 0 seconds [1706869.130543] LustreError: 137-5: fir-OST0019_UUID: not available for connect from 10.8.12.5@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706869.147912] LustreError: Skipped 99 previous similar messages [1706869.269858] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 1 seconds [1706871.269909] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 3 seconds [1706871.280255] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1706871.293656] Lustre: fir-OST0023: Imperative Recovery not enabled, recovery window 300-900 [1706871.302647] Lustre: fir-OST0023: in recovery but waiting for the first client to connect [1706871.339013] Lustre: fir-OST0023: Will be in recovery for at least 5:00, or until 511 clients reconnect [1706871.344386] Lustre: fir-OST0023: Connection restored to (at 10.8.2.3@o2ib6) [1706871.852392] Lustre: fir-OST0023: Connection restored to 3d8dfe83-25e5-247f-c731-de21f1e85c71 (at 10.8.12.10@o2ib6) [1706871.862913] Lustre: Skipped 15 previous similar messages [1706872.853708] Lustre: fir-OST0023: Connection restored to fb5e4f4f-a645-fbec-7b80-08c8d8a2fea0 (at 10.8.1.12@o2ib6) [1706872.864147] Lustre: Skipped 24 previous similar messages [1706874.904660] Lustre: fir-OST0023: Connection restored to (at 10.8.10.24@o2ib6) [1706874.912060] Lustre: Skipped 51 previous similar messages [1706875.270004] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 7 seconds [1706875.280347] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 2 previous similar messages [1706877.192599] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.10.6@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706877.210009] LustreError: Skipped 820 previous similar messages [1706878.205602] Lustre: fir-OST0023: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 511 known clients (175 recovered, 78 in progress, and 0 evicted) already passed deadline 0:06 [1706878.918376] Lustre: fir-OST0023: Connection restored to (at 10.8.7.31@o2ib6) [1706878.925692] Lustre: Skipped 200 previous similar messages [1706882.273043] Lustre: fir-OST0021: Imperative Recovery not enabled, recovery window 300-900 [1706882.294821] Lustre: fir-OST0021: in recovery but waiting for the first client to connect [1706882.307782] Lustre: fir-OST0021: Will be in recovery for at least 5:00, or until 514 clients reconnect [1706882.588435] Lustre: fir-OST001f: Not available for connect from 10.9.107.37@o2ib4 (not set up) [1706883.270193] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 2 seconds [1706883.280539] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 1 previous similar message [1706886.924754] Lustre: fir-OST0021: Connection restored to ed72929c-d604-56ff-ec0a-5b1f9da0af84 (at 10.8.17.9@o2ib6) [1706886.935192] Lustre: Skipped 480 previous similar messages [1706889.121539] Lustre: fir-OST0021: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 514 known clients (298 recovered, 133 in progress, and 0 evicted) already passed deadline 0:06 [1706892.273923] Lustre: fir-OST001f: Imperative Recovery not enabled, recovery window 300-900 [1706892.282983] Lustre: fir-OST001f: in recovery but waiting for the first client to connect [1706892.293146] Lustre: fir-OST001f: Will be in recovery for at least 5:00, or until 508 clients reconnect [1706892.656712] Lustre: fir-OST001b: Not available for connect from 10.8.25.26@o2ib6 (not set up) [1706892.853704] Lustre: fir-OST0023: Recovery over after 0:21, of 511 clients 511 recovered and 0 were evicted. [1706893.196649] LustreError: 137-5: fir-OST0018_UUID: not available for connect from 10.8.7.11@o2ib6 (no target). If you are running an HA pair check that the target is mounted on the other server. [1706893.214056] LustreError: Skipped 2710 previous similar messages [1706894.554679] Lustre: fir-OST0021: Recovery over after 0:12, of 514 clients 514 recovered and 0 were evicted. [1706897.273393] Lustre: fir-OST001b: Imperative Recovery not enabled, recovery window 300-900 [1706897.282399] Lustre: fir-OST001b: in recovery but waiting for the first client to connect [1706897.287619] Lustre: fir-OST001b: Will be in recovery for at least 5:00, or until 495 clients reconnect [1706898.899909] Lustre: fir-OST001f: Denying connection for new client 79496ef6-ef75-cc5f-9bd3-cf20df3c8e1d(at 10.8.25.27@o2ib6), waiting for 508 known clients (290 recovered, 128 in progress, and 0 evicted) already passed deadline 0:06 [1706898.920657] Lustre: Skipped 1 previous similar message [1706899.270558] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Timed out tx for 10.0.10.52@o2ib7: 10 seconds [1706899.280983] LNet: 118124:0:(o2iblnd_cb.c:3370:kiblnd_check_conns()) Skipped 8 previous similar messages [1706901.753390] Lustre: fir-OST001b: Denying connection for new client afd89a08-6b67-04f7-2303-d504e382d3cd(at 10.9.105.61@o2ib4), waiting for 495 known clients (190 recovered, 76 in progress, and 0 evicted) already passed deadline 0:04 [1706901.774168] Lustre: Skipped 2 previous similar messages [1706902.931732] Lustre: fir-OST001b: Connection restored to (at 10.8.18.23@o2ib6) [1706902.939146] Lustre: Skipped 1096 previous similar messages [1706907.273599] Lustre: fir-OST001d: Imperative Recovery not enabled, recovery window 300-900 [1706907.282458] Lustre: fir-OST001d: in recovery but waiting for the first client to connect [1706907.285265] Lustre: fir-OST001d: Will be in recovery for at least 5:00, or until 507 clients reconnect [1706911.193591] Lustre: fir-OST001f: Recovery over after 0:19, of 508 clients 508 recovered and 0 were evicted. [1706915.288860] Lustre: fir-OST0019: Denying connection for new client afd89a08-6b67-04f7-2303-d504e382d3cd(at 10.9.105.61@o2ib4), waiting for 557 known clients (158 recovered, 63 in progress, and 0 evicted) already passed deadline 0:03 [1706915.309615] Lustre: Skipped 2 previous similar messages [1706924.494456] Lustre: fir-OST001b: Recovery over after 0:27, of 495 clients 495 recovered and 0 were evicted. [1706936.382394] Lustre: fir-OST001b: Connection restored to (at 10.8.25.27@o2ib6) [1706936.389791] Lustre: Skipped 1256 previous similar messages [1706969.071450] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000400:148694 to 0xcc0000400:148737 [1706969.091555] Lustre: fir-OST0021: deleting orphan objects from 0x980000400:148694 to 0x980000400:148737 [1706969.096644] Lustre: fir-OST001b: deleting orphan objects from 0xd00000400:148590 to 0xd00000400:148609 [1706969.098089] Lustre: fir-OST0023: deleting orphan objects from 0xd40000400:148630 to 0xd40000400:148673 [1706969.104126] Lustre: fir-OST0019: deleting orphan objects from 0x640000401:148635 to 0x640000401:148673 [1706969.106938] Lustre: fir-OST001f: deleting orphan objects from 0x480000401:148735 to 0x480000401:148769 [1706969.130195] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000401:137928 to 0xcc0000401:137953 [1706969.139756] Lustre: fir-OST0023: deleting orphan objects from 0xd40000401:137970 to 0xd40000401:137985 [1706969.146858] Lustre: fir-OST001f: deleting orphan objects from 0x480000402:137923 to 0x480000402:137953 [1706969.151163] Lustre: fir-OST0019: deleting orphan objects from 0x640000402:137775 to 0x640000402:137793 [1706969.153135] Lustre: fir-OST0021: deleting orphan objects from 0x980000401:137926 to 0x980000401:137953 [1706969.160183] Lustre: fir-OST001b: deleting orphan objects from 0xd00000401:137780 to 0xd00000401:137825 [1707038.443467] Lustre: fir-OST0019: Connection restored to (at 10.9.103.31@o2ib4) [1707038.450969] Lustre: Skipped 79 previous similar messages [1707038.507303] Lustre: fir-OST0019: deleting orphan objects from 0x0:681818 to 0x0:681857 [1707038.507307] Lustre: fir-OST001f: deleting orphan objects from 0x0:681786 to 0x0:681825 [1707038.508077] Lustre: fir-OST001b: deleting orphan objects from 0x0:681736 to 0x0:681761 [1707038.514196] Lustre: fir-OST0021: deleting orphan objects from 0x0:681689 to 0x0:681729 [1707038.526406] Lustre: fir-OST001d: deleting orphan objects from 0x0:681801 to 0x0:681825 [1707038.547686] Lustre: fir-OST0023: deleting orphan objects from 0x0:681787 to 0x0:681825 [1707059.153386] Lustre: fir-OST0019: deleting orphan objects from 0x640000400:479273 to 0x640000400:479297 [1707059.153416] Lustre: fir-OST001d: deleting orphan objects from 0xcc0000402:479339 to 0xcc0000402:479361 [1707059.153419] Lustre: fir-OST0021: deleting orphan objects from 0x980000402:479243 to 0x980000402:479265 [1707059.153442] Lustre: fir-OST001f: deleting orphan objects from 0x480000400:479306 to 0x480000400:479329 [1707059.153448] Lustre: fir-OST001b: deleting orphan objects from 0xd00000402:479529 to 0xd00000402:479553 [1707059.186800] Lustre: fir-OST0023: deleting orphan objects from 0xd40000402:479241 to 0xd40000402:479265 [1707166.468064] Lustre: fir-OST001d: Connection restored to 03d523d0-d33d-b8d0-52c8-1ff235ea28e5 (at 10.9.102.16@o2ib4) [1707166.478672] Lustre: Skipped 1404 previous similar messages [1707422.691611] Lustre: fir-OST0023: Connection restored to 3ba4f6ab-da5b-8d5b-7e10-cb73b415cd02 (at 10.8.3.11@o2ib6) [1707422.702045] Lustre: Skipped 486 previous similar messages [1707937.833262] Lustre: fir-OST001b: Connection restored to (at 10.8.6.4@o2ib6) [1707937.840489] Lustre: Skipped 988 previous similar messages [1708539.740371] Lustre: fir-OST0021: Connection restored to 09366885-6351-1b6e-8051-f1297b8fedd9 (at 10.9.107.18@o2ib4) [1708539.750980] Lustre: Skipped 1010 previous similar messages [1708550.581820] LustreError: 123124:0:(tgt_grant.c:1079:tgt_grant_discard()) ASSERTION( tgd->tgd_tot_granted >= ted->ted_grant ) failed: fir-OST001b: tot_granted 50041695803 cli d5e4b60f-fe33-b991-7d48-5b8db7e07ab0/ffff926b10975c00 ted_grant -49152 [1708550.603611] LustreError: 123124:0:(tgt_grant.c:1079:tgt_grant_discard()) LBUG [1708550.610923] Pid: 123124, comm: ll_ost00_019 3.10.0-957.1.3.el7_lustre.x86_64 #1 SMP Fri Dec 7 14:50:35 PST 2018 [1708550.621180] Call Trace: [1708550.623814] [] libcfs_call_trace+0x8c/0xc0 [libcfs] [1708550.630548] [] lbug_with_loc+0x4c/0xa0 [libcfs] [1708550.636935] [] tgt_grant_discard+0x1dc/0x1e0 [ptlrpc] [1708550.643892] [] ofd_obd_disconnect+0x74/0x220 [ofd] [1708550.650541] [] target_handle_disconnect+0xd7/0x450 [ptlrpc] [1708550.658005] [] tgt_disconnect+0x37/0x140 [ptlrpc] [1708550.664609] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [1708550.671734] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1708550.679628] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [1708550.686136] [] kthread+0xd1/0xe0 [1708550.691224] [] ret_from_fork_nospec_begin+0xe/0x21 [1708550.697873] [] 0xffffffffffffffff [1708550.703065] Kernel panic - not syncing: LBUG [1708550.707509] CPU: 20 PID: 123124 Comm: ll_ost00_019 Kdump: loaded Tainted: G OE ------------ 3.10.0-957.1.3.el7_lustre.x86_64 #1 [1708550.720273] Hardware name: Dell Inc. PowerEdge R6415/065PKD, BIOS 1.6.7 10/29/2018 [1708550.728015] Call Trace: [1708550.730645] [] dump_stack+0x19/0x1b [1708550.735962] [] panic+0xe8/0x21f [1708550.740937] [] lbug_with_loc+0x9b/0xa0 [libcfs] [1708550.747346] [] tgt_grant_discard+0x1dc/0x1e0 [ptlrpc] [1708550.754230] [] ofd_obd_disconnect+0x74/0x220 [ofd] [1708550.760880] [] ? lustre_pack_reply+0x11/0x20 [ptlrpc] [1708550.767783] [] ? req_capsule_server_pack+0x43/0xf0 [ptlrpc] [1708550.775207] [] target_handle_disconnect+0xd7/0x450 [ptlrpc] [1708550.782634] [] tgt_disconnect+0x37/0x140 [ptlrpc] [1708550.789194] [] tgt_request_handle+0xaea/0x1580 [ptlrpc] [1708550.796272] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1708550.804022] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1708550.811281] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1708550.819142] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1708550.826110] [] ? default_wake_function+0x12/0x20 [1708550.832548] [] ? __wake_up_common+0x5b/0x90 [1708550.838589] [] ptlrpc_main+0xafc/0x1fc0 [ptlrpc] [1708550.845068] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1708550.852636] [] kthread+0xd1/0xe0 [1708550.857688] [] ? insert_kthread_work+0x40/0x40 [1708550.863956] [] ret_from_fork_nospec_begin+0xe/0x21 [1708550.870567] [] ? insert_kthread_work+0x40/0x40