[ 16.448831] scsi 2:0:97:0: enclosure logical id(0x5000ccab0405e080), slot(43) [ 16.456051] scsi 2:0:97:0: enclosure level(0x0000), connector name( C2 ) [ 16.462858] scsi 2:0:97:0: serial_number( 1SJRP9VZ) [ 16.468344] scsi 2:0:97:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 16.488536] mpt3sas_cm0: detecting: handle(0x0102), sas_address(0x5000cca2525596b1), phy(35) [ 16.496972] mpt3sas_cm0: REPORT_LUNS: handle(0x0102), retries(0) [ 16.503110] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0102), lun(0) [ 16.509774] scsi 2:0:98:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 16.518079] scsi 2:0:98:0: SSP: handle(0x0102), sas_addr(0x5000cca2525596b1), phy(35), device_name(0x5000cca2525596b3) [ 16.528765] scsi 2:0:98:0: enclosure logical id(0x5000ccab0405e080), slot(44) [ 16.535985] scsi 2:0:98:0: enclosure level(0x0000), connector name( C2 ) [ 16.542791] scsi 2:0:98:0: serial_number( 7SHJ25ZG) [ 16.548276] scsi 2:0:98:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 16.570539] mpt3sas_cm0: detecting: handle(0x0103), sas_address(0x5000cca2525593f5), phy(36) [ 16.578977] mpt3sas_cm0: REPORT_LUNS: handle(0x0103), retries(0) [ 16.585138] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0103), lun(0) [ 16.591786] scsi 2:0:99:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 16.600090] scsi 2:0:99:0: SSP: handle(0x0103), sas_addr(0x5000cca2525593f5), phy(36), device_name(0x5000cca2525593f7) [ 16.610776] scsi 2:0:99:0: enclosure logical id(0x5000ccab0405e080), slot(45) [ 16.617995] scsi 2:0:99:0: enclosure level(0x0000), connector name( C2 ) [ 16.624801] scsi 2:0:99:0: serial_number( 7SHJ20AG) [ 16.630288] scsi 2:0:99:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 16.652535] mpt3sas_cm0: detecting: handle(0x0104), sas_address(0x5000cca252541a05), phy(37) [ 16.660970] mpt3sas_cm0: REPORT_LUNS: handle(0x0104), retries(0) [ 16.667132] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0104), lun(0) [ 16.673842] scsi 2:0:100:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 16.682232] scsi 2:0:100:0: SSP: handle(0x0104), sas_addr(0x5000cca252541a05), phy(37), device_name(0x5000cca252541a07) [ 16.693005] scsi 2:0:100:0: enclosure logical id(0x5000ccab0405e080), slot(46) [ 16.700313] scsi 2:0:100:0: enclosure level(0x0000), connector name( C2 ) [ 16.707206] scsi 2:0:100:0: serial_number( 7SHH7V2G) [ 16.712777] scsi 2:0:100:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 16.735541] mpt3sas_cm0: detecting: handle(0x0105), sas_address(0x5000cca252561aa9), phy(38) [ 16.743980] mpt3sas_cm0: REPORT_LUNS: handle(0x0105), retries(0) [ 16.750145] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0105), lun(0) [ 16.756866] scsi 2:0:101:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 16.765252] scsi 2:0:101:0: SSP: handle(0x0105), sas_addr(0x5000cca252561aa9), phy(38), device_name(0x5000cca252561aab) [ 16.776024] scsi 2:0:101:0: enclosure logical id(0x5000ccab0405e080), slot(47) [ 16.783328] scsi 2:0:101:0: enclosure level(0x0000), connector name( C2 ) [ 16.790223] scsi 2:0:101:0: serial_number( 7SHJAZEG) [ 16.795795] scsi 2:0:101:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 16.818540] mpt3sas_cm0: detecting: handle(0x0106), sas_address(0x5000cca26b8e5a45), phy(39) [ 16.826982] mpt3sas_cm0: REPORT_LUNS: handle(0x0106), retries(0) [ 16.833132] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0106), lun(0) [ 16.839798] scsi 2:0:102:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 16.848199] scsi 2:0:102:0: SSP: handle(0x0106), sas_addr(0x5000cca26b8e5a45), phy(39), device_name(0x5000cca26b8e5a47) [ 16.858973] scsi 2:0:102:0: enclosure logical id(0x5000ccab0405e080), slot(48) [ 16.866279] scsi 2:0:102:0: enclosure level(0x0000), connector name( C2 ) [ 16.873170] scsi 2:0:102:0: serial_number( 1SJJ8YNZ) [ 16.878743] scsi 2:0:102:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 16.898539] mpt3sas_cm0: detecting: handle(0x0107), sas_address(0x5000cca26b9bf6c1), phy(40) [ 16.906975] mpt3sas_cm0: REPORT_LUNS: handle(0x0107), retries(0) [ 16.913108] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0107), lun(0) [ 16.919740] scsi 2:0:103:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 16.928132] scsi 2:0:103:0: SSP: handle(0x0107), sas_addr(0x5000cca26b9bf6c1), phy(40), device_name(0x5000cca26b9bf6c3) [ 16.938905] scsi 2:0:103:0: enclosure logical id(0x5000ccab0405e080), slot(49) [ 16.946210] scsi 2:0:103:0: enclosure level(0x0000), connector name( C2 ) [ 16.953104] scsi 2:0:103:0: serial_number( 1SJST0EZ) [ 16.958676] scsi 2:0:103:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 16.978538] mpt3sas_cm0: detecting: handle(0x0108), sas_address(0x5000cca252551575), phy(41) [ 16.986977] mpt3sas_cm0: REPORT_LUNS: handle(0x0108), retries(0) [ 16.993128] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0108), lun(0) [ 16.999797] scsi 2:0:104:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.008191] scsi 2:0:104:0: SSP: handle(0x0108), sas_addr(0x5000cca252551575), phy(41), device_name(0x5000cca252551577) [ 17.018967] scsi 2:0:104:0: enclosure logical id(0x5000ccab0405e080), slot(50) [ 17.026272] scsi 2:0:104:0: enclosure level(0x0000), connector name( C2 ) [ 17.033166] scsi 2:0:104:0: serial_number( 7SHHTL5G) [ 17.038740] scsi 2:0:104:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.067542] mpt3sas_cm0: detecting: handle(0x0109), sas_address(0x5000cca25254ebe9), phy(42) [ 17.076000] mpt3sas_cm0: REPORT_LUNS: handle(0x0109), retries(0) [ 17.082135] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0109), lun(0) [ 17.088799] scsi 2:0:105:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.097188] scsi 2:0:105:0: SSP: handle(0x0109), sas_addr(0x5000cca25254ebe9), phy(42), device_name(0x5000cca25254ebeb) [ 17.107957] scsi 2:0:105:0: enclosure logical id(0x5000ccab0405e080), slot(51) [ 17.115262] scsi 2:0:105:0: enclosure level(0x0000), connector name( C2 ) [ 17.122156] scsi 2:0:105:0: serial_number( 7SHHPUDG) [ 17.127727] scsi 2:0:105:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.147548] mpt3sas_cm0: detecting: handle(0x010a), sas_address(0x5000cca2525595f1), phy(43) [ 17.155986] mpt3sas_cm0: REPORT_LUNS: handle(0x010a), retries(0) [ 17.162154] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010a), lun(0) [ 17.168798] scsi 2:0:106:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.177181] scsi 2:0:106:0: SSP: handle(0x010a), sas_addr(0x5000cca2525595f1), phy(43), device_name(0x5000cca2525595f3) [ 17.187951] scsi 2:0:106:0: enclosure logical id(0x5000ccab0405e080), slot(52) [ 17.195256] scsi 2:0:106:0: enclosure level(0x0000), connector name( C2 ) [ 17.202149] scsi 2:0:106:0: serial_number( 7SHJ24EG) [ 17.207722] scsi 2:0:106:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.228132] mpt3sas_cm0: detecting: handle(0x010b), sas_address(0x5000cca26b9c1ee1), phy(44) [ 17.236567] mpt3sas_cm0: REPORT_LUNS: handle(0x010b), retries(0) [ 17.242740] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010b), lun(0) [ 17.249391] scsi 2:0:107:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.257776] scsi 2:0:107:0: SSP: handle(0x010b), sas_addr(0x5000cca26b9c1ee1), phy(44), device_name(0x5000cca26b9c1ee3) [ 17.268551] scsi 2:0:107:0: enclosure logical id(0x5000ccab0405e080), slot(53) [ 17.275856] scsi 2:0:107:0: enclosure level(0x0000), connector name( C2 ) [ 17.282747] scsi 2:0:107:0: serial_number( 1SJSVP8Z) [ 17.288320] scsi 2:0:107:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.308542] mpt3sas_cm0: detecting: handle(0x010c), sas_address(0x5000cca26b9e296d), phy(45) [ 17.316986] mpt3sas_cm0: REPORT_LUNS: handle(0x010c), retries(0) [ 17.323128] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010c), lun(0) [ 17.329761] scsi 2:0:108:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.338150] scsi 2:0:108:0: SSP: handle(0x010c), sas_addr(0x5000cca26b9e296d), phy(45), device_name(0x5000cca26b9e296f) [ 17.348923] scsi 2:0:108:0: enclosure logical id(0x5000ccab0405e080), slot(54) [ 17.356231] scsi 2:0:108:0: enclosure level(0x0000), connector name( C2 ) [ 17.363123] scsi 2:0:108:0: serial_number( 1SJTZH2Z) [ 17.368695] scsi 2:0:108:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.388540] mpt3sas_cm0: detecting: handle(0x010d), sas_address(0x5000cca26a264599), phy(46) [ 17.396978] mpt3sas_cm0: REPORT_LUNS: handle(0x010d), retries(0) [ 17.403118] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010d), lun(0) [ 17.409760] scsi 2:0:109:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.418170] scsi 2:0:109:0: SSP: handle(0x010d), sas_addr(0x5000cca26a264599), phy(46), device_name(0x5000cca26a26459b) [ 17.428943] scsi 2:0:109:0: enclosure logical id(0x5000ccab0405e080), slot(55) [ 17.436248] scsi 2:0:109:0: enclosure level(0x0000), connector name( C2 ) [ 17.443143] scsi 2:0:109:0: serial_number( 2TGP1H9D) [ 17.448715] scsi 2:0:109:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.487539] mpt3sas_cm0: detecting: handle(0x010e), sas_address(0x5000cca26b9a83ad), phy(47) [ 17.495990] mpt3sas_cm0: REPORT_LUNS: handle(0x010e), retries(0) [ 17.502114] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010e), lun(0) [ 17.508782] scsi 2:0:110:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.517173] scsi 2:0:110:0: SSP: handle(0x010e), sas_addr(0x5000cca26b9a83ad), phy(47), device_name(0x5000cca26b9a83af) [ 17.527942] scsi 2:0:110:0: enclosure logical id(0x5000ccab0405e080), slot(56) [ 17.535249] scsi 2:0:110:0: enclosure level(0x0000), connector name( C2 ) [ 17.542141] scsi 2:0:110:0: serial_number( 1SJRZ9AZ) [ 17.547716] scsi 2:0:110:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.567551] mpt3sas_cm0: detecting: handle(0x010f), sas_address(0x5000cca26b9ccd89), phy(48) [ 17.575989] mpt3sas_cm0: REPORT_LUNS: handle(0x010f), retries(0) [ 17.582157] mpt3sas_cm0: TEST_UNIT_READY: handle(0x010f), lun(0) [ 17.588824] scsi 2:0:111:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.597215] scsi 2:0:111:0: SSP: handle(0x010f), sas_addr(0x5000cca26b9ccd89), phy(48), device_name(0x5000cca26b9ccd8b) [ 17.607988] scsi 2:0:111:0: enclosure logical id(0x5000ccab0405e080), slot(57) [ 17.615293] scsi 2:0:111:0: enclosure level(0x0000), connector name( C2 ) [ 17.622187] scsi 2:0:111:0: serial_number( 1SJT79VZ) [ 17.627760] scsi 2:0:111:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.647551] mpt3sas_cm0: detecting: handle(0x0110), sas_address(0x5000cca26b9b9705), phy(49) [ 17.655989] mpt3sas_cm0: REPORT_LUNS: handle(0x0110), retries(0) [ 17.662136] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0110), lun(0) [ 17.864137] scsi 2:0:112:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.872544] scsi 2:0:112:0: SSP: handle(0x0110), sas_addr(0x5000cca26b9b9705), phy(49), device_name(0x5000cca26b9b9707) [ 17.883319] scsi 2:0:112:0: enclosure logical id(0x5000ccab0405e080), slot(58) [ 17.890624] scsi 2:0:112:0: enclosure level(0x0000), connector name( C2 ) [ 17.897502] scsi 2:0:112:0: serial_number( 1SJSKMTZ) [ 17.903081] scsi 2:0:112:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 17.943548] mpt3sas_cm0: detecting: handle(0x0111), sas_address(0x5000cca25253f3a5), phy(50) [ 17.951988] mpt3sas_cm0: REPORT_LUNS: handle(0x0111), retries(0) [ 17.958129] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0111), lun(0) [ 17.964898] scsi 2:0:113:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 17.973288] scsi 2:0:113:0: SSP: handle(0x0111), sas_addr(0x5000cca25253f3a5), phy(50), device_name(0x5000cca25253f3a7) [ 17.984059] scsi 2:0:113:0: enclosure logical id(0x5000ccab0405e080), slot(59) [ 17.991365] scsi 2:0:113:0: enclosure level(0x0000), connector name( C2 ) [ 17.998259] scsi 2:0:113:0: serial_number( 7SHH58VG) [ 18.003831] scsi 2:0:113:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.042035] mpt3sas_cm0: expander_add: handle(0x00dd), parent(0x00da), sas_addr(0x5000ccab0405e0ff), phys(68) [ 18.063017] mpt3sas_cm0: detecting: handle(0x0112), sas_address(0x5000cca2525629fd), phy(42) [ 18.071461] mpt3sas_cm0: REPORT_LUNS: handle(0x0112), retries(0) [ 18.077586] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0112), lun(0) [ 18.084228] scsi 2:0:114:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.092625] scsi 2:0:114:0: SSP: handle(0x0112), sas_addr(0x5000cca2525629fd), phy(42), device_name(0x5000cca2525629ff) [ 18.103398] scsi 2:0:114:0: enclosure logical id(0x5000ccab0405e080), slot(1) [ 18.110619] scsi 2:0:114:0: enclosure level(0x0000), connector name( C2 ) [ 18.117509] scsi 2:0:114:0: serial_number( 7SHJD02G) [ 18.123084] scsi 2:0:114:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.145550] mpt3sas_cm0: detecting: handle(0x0113), sas_address(0x5000cca26a2a8505), phy(43) [ 18.153982] mpt3sas_cm0: REPORT_LUNS: handle(0x0113), retries(0) [ 18.160301] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0113), lun(0) [ 18.179467] scsi 2:0:115:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.187851] scsi 2:0:115:0: SSP: handle(0x0113), sas_addr(0x5000cca26a2a8505), phy(43), device_name(0x5000cca26a2a8507) [ 18.198618] scsi 2:0:115:0: enclosure logical id(0x5000ccab0405e080), slot(3) [ 18.205839] scsi 2:0:115:0: enclosure level(0x0000), connector name( C2 ) [ 18.212717] scsi 2:0:115:0: serial_number( 2TGSBX9D) [ 18.218297] scsi 2:0:115:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.242151] mpt3sas_cm0: detecting: handle(0x0114), sas_address(0x5000cca252555741), phy(44) [ 18.250592] mpt3sas_cm0: REPORT_LUNS: handle(0x0114), retries(0) [ 18.256735] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0114), lun(0) [ 18.263399] scsi 2:0:116:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.271793] scsi 2:0:116:0: SSP: handle(0x0114), sas_addr(0x5000cca252555741), phy(44), device_name(0x5000cca252555743) [ 18.282566] scsi 2:0:116:0: enclosure logical id(0x5000ccab0405e080), slot(4) [ 18.289783] scsi 2:0:116:0: enclosure level(0x0000), connector name( C2 ) [ 18.296678] scsi 2:0:116:0: serial_number( 7SHHXZ0G) [ 18.302249] scsi 2:0:116:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.322555] mpt3sas_cm0: detecting: handle(0x0115), sas_address(0x5000cca26a24838d), phy(45) [ 18.330992] mpt3sas_cm0: REPORT_LUNS: handle(0x0115), retries(0) [ 18.337162] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0115), lun(0) [ 18.343813] scsi 2:0:117:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.352200] scsi 2:0:117:0: SSP: handle(0x0115), sas_addr(0x5000cca26a24838d), phy(45), device_name(0x5000cca26a24838f) [ 18.362973] scsi 2:0:117:0: enclosure logical id(0x5000ccab0405e080), slot(5) [ 18.370194] scsi 2:0:117:0: enclosure level(0x0000), connector name( C2 ) [ 18.377085] scsi 2:0:117:0: serial_number( 2TGN2J5D) [ 18.382658] scsi 2:0:117:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.402553] mpt3sas_cm0: detecting: handle(0x0116), sas_address(0x5000cca25256134d), phy(46) [ 18.410994] mpt3sas_cm0: REPORT_LUNS: handle(0x0116), retries(0) [ 18.417132] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0116), lun(0) [ 18.423786] scsi 2:0:118:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.432179] scsi 2:0:118:0: SSP: handle(0x0116), sas_addr(0x5000cca25256134d), phy(46), device_name(0x5000cca25256134f) [ 18.442951] scsi 2:0:118:0: enclosure logical id(0x5000ccab0405e080), slot(6) [ 18.450170] scsi 2:0:118:0: enclosure level(0x0000), connector name( C2 ) [ 18.457062] scsi 2:0:118:0: serial_number( 7SHJAH7G) [ 18.462635] scsi 2:0:118:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.487571] mpt3sas_cm0: detecting: handle(0x0117), sas_address(0x5000cca26a2b4099), phy(47) [ 18.496028] mpt3sas_cm0: REPORT_LUNS: handle(0x0117), retries(0) [ 18.502147] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0117), lun(0) [ 18.508782] scsi 2:0:119:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.517171] scsi 2:0:119:0: SSP: handle(0x0117), sas_addr(0x5000cca26a2b4099), phy(47), device_name(0x5000cca26a2b409b) [ 18.527945] scsi 2:0:119:0: enclosure logical id(0x5000ccab0405e080), slot(7) [ 18.535163] scsi 2:0:119:0: enclosure level(0x0000), connector name( C2 ) [ 18.542058] scsi 2:0:119:0: serial_number( 2TGSTDKD) [ 18.547631] scsi 2:0:119:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.567556] mpt3sas_cm0: detecting: handle(0x0118), sas_address(0x5000cca26a2b9365), phy(48) [ 18.575999] mpt3sas_cm0: REPORT_LUNS: handle(0x0118), retries(0) [ 18.582138] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0118), lun(0) [ 18.588830] scsi 2:0:120:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.597215] scsi 2:0:120:0: SSP: handle(0x0118), sas_addr(0x5000cca26a2b9365), phy(48), device_name(0x5000cca26a2b9367) [ 18.607989] scsi 2:0:120:0: enclosure logical id(0x5000ccab0405e080), slot(8) [ 18.615208] scsi 2:0:120:0: enclosure level(0x0000), connector name( C2 ) [ 18.622100] scsi 2:0:120:0: serial_number( 2TGSYXHD) [ 18.627675] scsi 2:0:120:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.647556] mpt3sas_cm0: detecting: handle(0x0119), sas_address(0x5000cca2525639f5), phy(49) [ 18.655991] mpt3sas_cm0: REPORT_LUNS: handle(0x0119), retries(0) [ 18.662742] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0119), lun(0) [ 18.670389] scsi 2:0:121:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.678788] scsi 2:0:121:0: SSP: handle(0x0119), sas_addr(0x5000cca2525639f5), phy(49), device_name(0x5000cca2525639f7) [ 18.689560] scsi 2:0:121:0: enclosure logical id(0x5000ccab0405e080), slot(9) [ 18.696780] scsi 2:0:121:0: enclosure level(0x0000), connector name( C2 ) [ 18.703673] scsi 2:0:121:0: serial_number( 7SHJE21G) [ 18.709245] scsi 2:0:121:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.729566] mpt3sas_cm0: detecting: handle(0x011a), sas_address(0x5000cca26a29c989), phy(50) [ 18.738004] mpt3sas_cm0: REPORT_LUNS: handle(0x011a), retries(0) [ 18.744146] mpt3sas_cm0: TEST_UNIT_READY: handle(0x011a), lun(0) [ 18.750796] scsi 2:0:122:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 18.759189] scsi 2:0:122:0: SSP: handle(0x011a), sas_addr(0x5000cca26a29c989), phy(50), device_name(0x5000cca26a29c98b) [ 18.769961] scsi 2:0:122:0: enclosure logical id(0x5000ccab0405e080), slot(10) [ 18.777266] scsi 2:0:122:0: enclosure level(0x0000), connector name( C2 ) [ 18.784158] scsi 2:0:122:0: serial_number( 2TGRZE7D) [ 18.789733] scsi 2:0:122:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.815074] mpt3sas_cm0: expander_add: handle(0x0017), parent(0x0009), sas_addr(0x5000ccab0405dcfd), phys(49) [ 18.835925] mpt3sas_cm0: detecting: handle(0x001b), sas_address(0x5000ccab0405dcfc), phy(48) [ 18.844363] mpt3sas_cm0: REPORT_LUNS: handle(0x001b), retries(0) [ 18.850718] mpt3sas_cm0: TEST_UNIT_READY: handle(0x001b), lun(0) [ 18.857596] scsi 2:0:123:0: Enclosure HGST H4060-J 2033 PQ: 0 ANSI: 6 [ 18.866196] scsi 2:0:123:0: set ignore_delay_remove for handle(0x001b) [ 18.872721] scsi 2:0:123:0: SES: handle(0x001b), sas_addr(0x5000ccab0405dcfc), phy(48), device_name(0x0000000000000000) [ 18.883493] scsi 2:0:123:0: enclosure logical id(0x5000ccab0405dc80), slot(60) [ 18.890799] scsi 2:0:123:0: enclosure level(0x0000), connector name( C1 ) [ 18.897692] scsi 2:0:123:0: serial_number(USWSJ03918EZ0068 ) [ 18.903612] scsi 2:0:123:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 18.924995] mpt3sas_cm0: expander_add: handle(0x0019), parent(0x0017), sas_addr(0x5000ccab0405dcf9), phys(68) [ 18.946635] mpt3sas_cm0: detecting: handle(0x001c), sas_address(0x5000cca252561066), phy(0) [ 18.954989] mpt3sas_cm0: REPORT_LUNS: handle(0x001c), retries(0) [ 18.961111] mpt3sas_cm0: TEST_UNIT_READY: handle(0x001c), lun(0) [ 19.001144] scsi 2:0:124:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.009532] scsi 2:0:124:0: SSP: handle(0x001c), sas_addr(0x5000cca252561066), phy(0), device_name(0x5000cca252561067) [ 19.020219] scsi 2:0:124:0: enclosure logical id(0x5000ccab0405dc80), slot(0) [ 19.027437] scsi 2:0:124:0: enclosure level(0x0000), connector name( C1 ) [ 19.034331] scsi 2:0:124:0: serial_number( 7SHJA97G) [ 19.039906] scsi 2:0:124:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.087653] mpt3sas_cm0: detecting: handle(0x001d), sas_address(0x5000cca25253f346), phy(1) [ 19.096004] mpt3sas_cm0: REPORT_LUNS: handle(0x001d), retries(0) [ 19.102161] mpt3sas_cm0: TEST_UNIT_READY: handle(0x001d), lun(0) [ 19.130039] scsi 2:0:125:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.138419] scsi 2:0:125:0: SSP: handle(0x001d), sas_addr(0x5000cca25253f346), phy(1), device_name(0x5000cca25253f347) [ 19.149101] scsi 2:0:125:0: enclosure logical id(0x5000ccab0405dc80), slot(2) [ 19.156318] scsi 2:0:125:0: enclosure level(0x0000), connector name( C1 ) [ 19.163196] scsi 2:0:125:0: serial_number( 7SHH582G) [ 19.168767] scsi 2:0:125:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.188552] mpt3sas_cm0: detecting: handle(0x001e), sas_address(0x5000cca252543b52), phy(2) [ 19.196900] mpt3sas_cm0: REPORT_LUNS: handle(0x001e), retries(0) [ 19.203057] mpt3sas_cm0: TEST_UNIT_READY: handle(0x001e), lun(0) [ 19.209795] scsi 2:0:126:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.218190] scsi 2:0:126:0: SSP: handle(0x001e), sas_addr(0x5000cca252543b52), phy(2), device_name(0x5000cca252543b53) [ 19.228876] scsi 2:0:126:0: enclosure logical id(0x5000ccab0405dc80), slot(11) [ 19.236183] scsi 2:0:126:0: enclosure level(0x0000), connector name( C1 ) [ 19.243076] scsi 2:0:126:0: serial_number( 7SHHA1UG) [ 19.248648] scsi 2:0:126:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.268549] mpt3sas_cm0: detecting: handle(0x001f), sas_address(0x5000cca252559f4e), phy(3) [ 19.276904] mpt3sas_cm0: REPORT_LUNS: handle(0x001f), retries(0) [ 19.283046] mpt3sas_cm0: TEST_UNIT_READY: handle(0x001f), lun(0) [ 19.310806] scsi 2:0:127:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.319253] scsi 2:0:127:0: SSP: handle(0x001f), sas_addr(0x5000cca252559f4e), phy(3), device_name(0x5000cca252559f4f) [ 19.329940] scsi 2:0:127:0: enclosure logical id(0x5000ccab0405dc80), slot(12) [ 19.337245] scsi 2:0:127:0: enclosure level(0x0000), connector name( C1 ) [ 19.344137] scsi 2:0:127:0: serial_number( 7SHJ2SSG) [ 19.349711] scsi 2:0:127:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.378664] mpt3sas_cm0: detecting: handle(0x0020), sas_address(0x5000cca252544e4e), phy(4) [ 19.387016] mpt3sas_cm0: REPORT_LUNS: handle(0x0020), retries(0) [ 19.393170] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0020), lun(0) [ 19.400767] scsi 2:0:128:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.409592] scsi 2:0:128:0: SSP: handle(0x0020), sas_addr(0x5000cca252544e4e), phy(4), device_name(0x5000cca252544e4f) [ 19.420281] scsi 2:0:128:0: enclosure logical id(0x5000ccab0405dc80), slot(13) [ 19.427586] scsi 2:0:128:0: enclosure level(0x0000), connector name( C1 ) [ 19.434479] scsi 2:0:128:0: serial_number( 7SHHBA0G) [ 19.440051] scsi 2:0:128:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.468553] mpt3sas_cm0: detecting: handle(0x0021), sas_address(0x5000cca25253e40e), phy(5) [ 19.476919] mpt3sas_cm0: REPORT_LUNS: handle(0x0021), retries(0) [ 19.483072] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0021), lun(0) [ 19.496018] scsi 2:0:129:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.504477] scsi 2:0:129:0: SSP: handle(0x0021), sas_addr(0x5000cca25253e40e), phy(5), device_name(0x5000cca25253e40f) [ 19.515163] scsi 2:0:129:0: enclosure logical id(0x5000ccab0405dc80), slot(14) [ 19.522468] scsi 2:0:129:0: enclosure level(0x0000), connector name( C1 ) [ 19.529346] scsi 2:0:129:0: serial_number( 7SHH47NG) [ 19.534925] scsi 2:0:129:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.570585] mpt3sas_cm0: detecting: handle(0x0022), sas_address(0x5000cca26b9adc2a), phy(6) [ 19.578935] mpt3sas_cm0: REPORT_LUNS: handle(0x0022), retries(0) [ 19.585106] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0022), lun(0) [ 19.592033] scsi 2:0:130:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.612979] scsi 2:0:130:0: SSP: handle(0x0022), sas_addr(0x5000cca26b9adc2a), phy(6), device_name(0x5000cca26b9adc2b) [ 19.623669] scsi 2:0:130:0: enclosure logical id(0x5000ccab0405dc80), slot(15) [ 19.630973] scsi 2:0:130:0: enclosure level(0x0000), connector name( C1 ) [ 19.637867] scsi 2:0:130:0: serial_number( 1SJS560Z) [ 19.643442] scsi 2:0:130:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.683579] mpt3sas_cm0: detecting: handle(0x0023), sas_address(0x5000cca252550a7e), phy(7) [ 19.691931] mpt3sas_cm0: REPORT_LUNS: handle(0x0023), retries(0) [ 19.698087] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0023), lun(0) [ 19.704844] scsi 2:0:131:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.713236] scsi 2:0:131:0: SSP: handle(0x0023), sas_addr(0x5000cca252550a7e), phy(7), device_name(0x5000cca252550a7f) [ 19.723924] scsi 2:0:131:0: enclosure logical id(0x5000ccab0405dc80), slot(16) [ 19.731230] scsi 2:0:131:0: enclosure level(0x0000), connector name( C1 ) [ 19.738125] scsi 2:0:131:0: serial_number( 7SHHSVJG) [ 19.743695] scsi 2:0:131:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.763547] mpt3sas_cm0: detecting: handle(0x0024), sas_address(0x5000cca25253fc92), phy(8) [ 19.771900] mpt3sas_cm0: REPORT_LUNS: handle(0x0024), retries(0) [ 19.778063] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0024), lun(0) [ 19.800491] scsi 2:0:132:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.808882] scsi 2:0:132:0: SSP: handle(0x0024), sas_addr(0x5000cca25253fc92), phy(8), device_name(0x5000cca25253fc93) [ 19.819570] scsi 2:0:132:0: enclosure logical id(0x5000ccab0405dc80), slot(17) [ 19.826876] scsi 2:0:132:0: enclosure level(0x0000), connector name( C1 ) [ 19.833753] scsi 2:0:132:0: serial_number( 7SHH5W8G) [ 19.839332] scsi 2:0:132:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.859556] mpt3sas_cm0: detecting: handle(0x0025), sas_address(0x5000cca26b9d228a), phy(9) [ 19.867907] mpt3sas_cm0: REPORT_LUNS: handle(0x0025), retries(0) [ 19.874071] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0025), lun(0) [ 19.881057] scsi 2:0:133:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.898956] scsi 2:0:133:0: SSP: handle(0x0025), sas_addr(0x5000cca26b9d228a), phy(9), device_name(0x5000cca26b9d228b) [ 19.909641] scsi 2:0:133:0: enclosure logical id(0x5000ccab0405dc80), slot(18) [ 19.916947] scsi 2:0:133:0: enclosure level(0x0000), connector name( C1 ) [ 19.923838] scsi 2:0:133:0: serial_number( 1SJTDZAZ) [ 19.929414] scsi 2:0:133:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 19.952553] mpt3sas_cm0: detecting: handle(0x0026), sas_address(0x5000cca2525448f6), phy(10) [ 19.960986] mpt3sas_cm0: REPORT_LUNS: handle(0x0026), retries(0) [ 19.967146] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0026), lun(0) [ 19.991035] scsi 2:0:134:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 19.999422] scsi 2:0:134:0: SSP: handle(0x0026), sas_addr(0x5000cca2525448f6), phy(10), device_name(0x5000cca2525448f7) [ 20.010192] scsi 2:0:134:0: enclosure logical id(0x5000ccab0405dc80), slot(19) [ 20.017499] scsi 2:0:134:0: enclosure level(0x0000), connector name( C1 ) [ 20.024377] scsi 2:0:134:0: serial_number( 7SHHAYZG) [ 20.029955] scsi 2:0:134:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.052586] mpt3sas_cm0: detecting: handle(0x0027), sas_address(0x5000cca26a234e2e), phy(11) [ 20.061026] mpt3sas_cm0: REPORT_LUNS: handle(0x0027), retries(0) [ 20.067161] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0027), lun(0) [ 20.073811] scsi 2:0:135:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.082202] scsi 2:0:135:0: SSP: handle(0x0027), sas_addr(0x5000cca26a234e2e), phy(11), device_name(0x5000cca26a234e2f) [ 20.092978] scsi 2:0:135:0: enclosure logical id(0x5000ccab0405dc80), slot(20) [ 20.100283] scsi 2:0:135:0: enclosure level(0x0000), connector name( C1 ) [ 20.107173] scsi 2:0:135:0: serial_number( 2TGMDXGD) [ 20.112748] scsi 2:0:135:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.132570] mpt3sas_cm0: detecting: handle(0x0028), sas_address(0x5000cca26a220396), phy(12) [ 20.141012] mpt3sas_cm0: REPORT_LUNS: handle(0x0028), retries(0) [ 20.147151] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0028), lun(0) [ 20.153799] scsi 2:0:136:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.162218] scsi 2:0:136:0: SSP: handle(0x0028), sas_addr(0x5000cca26a220396), phy(12), device_name(0x5000cca26a220397) [ 20.172986] scsi 2:0:136:0: enclosure logical id(0x5000ccab0405dc80), slot(21) [ 20.180293] scsi 2:0:136:0: enclosure level(0x0000), connector name( C1 ) [ 20.187185] scsi 2:0:136:0: serial_number( 2TGLPWYD) [ 20.192759] scsi 2:0:136:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.212553] mpt3sas_cm0: detecting: handle(0x0029), sas_address(0x5000cca26a228cb2), phy(13) [ 20.220986] mpt3sas_cm0: REPORT_LUNS: handle(0x0029), retries(0) [ 20.227117] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0029), lun(0) [ 20.233785] scsi 2:0:137:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.242172] scsi 2:0:137:0: SSP: handle(0x0029), sas_addr(0x5000cca26a228cb2), phy(13), device_name(0x5000cca26a228cb3) [ 20.252944] scsi 2:0:137:0: enclosure logical id(0x5000ccab0405dc80), slot(22) [ 20.260250] scsi 2:0:137:0: enclosure level(0x0000), connector name( C1 ) [ 20.267145] scsi 2:0:137:0: serial_number( 2TGM010D) [ 20.272717] scsi 2:0:137:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.292558] mpt3sas_cm0: detecting: handle(0x002a), sas_address(0x5000cca26a2223ee), phy(14) [ 20.300999] mpt3sas_cm0: REPORT_LUNS: handle(0x002a), retries(0) [ 20.307138] mpt3sas_cm0: TEST_UNIT_READY: handle(0x002a), lun(0) [ 20.313809] scsi 2:0:138:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.322201] scsi 2:0:138:0: SSP: handle(0x002a), sas_addr(0x5000cca26a2223ee), phy(14), device_name(0x5000cca26a2223ef) [ 20.332972] scsi 2:0:138:0: enclosure logical id(0x5000ccab0405dc80), slot(23) [ 20.340278] scsi 2:0:138:0: enclosure level(0x0000), connector name( C1 ) [ 20.347173] scsi 2:0:138:0: serial_number( 2TGLT1RD) [ 20.352743] scsi 2:0:138:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.372556] mpt3sas_cm0: detecting: handle(0x002b), sas_address(0x5000cca26a288e36), phy(15) [ 20.380993] mpt3sas_cm0: REPORT_LUNS: handle(0x002b), retries(0) [ 20.387138] mpt3sas_cm0: TEST_UNIT_READY: handle(0x002b), lun(0) [ 20.393853] scsi 2:0:139:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.402244] scsi 2:0:139:0: SSP: handle(0x002b), sas_addr(0x5000cca26a288e36), phy(15), device_name(0x5000cca26a288e37) [ 20.413019] scsi 2:0:139:0: enclosure logical id(0x5000ccab0405dc80), slot(24) [ 20.420324] scsi 2:0:139:0: enclosure level(0x0000), connector name( C1 ) [ 20.427215] scsi 2:0:139:0: serial_number( 2TGR9E7D) [ 20.432789] scsi 2:0:139:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.455557] mpt3sas_cm0: detecting: handle(0x002c), sas_address(0x5000cca26a24abfe), phy(16) [ 20.463995] mpt3sas_cm0: REPORT_LUNS: handle(0x002c), retries(0) [ 20.470158] mpt3sas_cm0: TEST_UNIT_READY: handle(0x002c), lun(0) [ 20.483445] scsi 2:0:140:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.491827] scsi 2:0:140:0: SSP: handle(0x002c), sas_addr(0x5000cca26a24abfe), phy(16), device_name(0x5000cca26a24abff) [ 20.502598] scsi 2:0:140:0: enclosure logical id(0x5000ccab0405dc80), slot(25) [ 20.509903] scsi 2:0:140:0: enclosure level(0x0000), connector name( C1 ) [ 20.516794] scsi 2:0:140:0: serial_number( 2TGN56ND) [ 20.522370] scsi 2:0:140:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.572574] mpt3sas_cm0: detecting: handle(0x002d), sas_address(0x5000cca26a26cb06), phy(17) [ 20.581010] mpt3sas_cm0: REPORT_LUNS: handle(0x002d), retries(0) [ 20.587188] mpt3sas_cm0: TEST_UNIT_READY: handle(0x002d), lun(0) [ 20.613699] scsi 2:0:141:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.622080] scsi 2:0:141:0: SSP: handle(0x002d), sas_addr(0x5000cca26a26cb06), phy(17), device_name(0x5000cca26a26cb07) [ 20.632847] scsi 2:0:141:0: enclosure logical id(0x5000ccab0405dc80), slot(26) [ 20.640154] scsi 2:0:141:0: enclosure level(0x0000), connector name( C1 ) [ 20.647029] scsi 2:0:141:0: serial_number( 2TGPABSD) [ 20.652600] scsi 2:0:141:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.672592] mpt3sas_cm0: detecting: handle(0x002e), sas_address(0x5000cca26a27af2a), phy(18) [ 20.681030] mpt3sas_cm0: REPORT_LUNS: handle(0x002e), retries(0) [ 20.687188] mpt3sas_cm0: TEST_UNIT_READY: handle(0x002e), lun(0) [ 20.713667] scsi 2:0:142:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.722036] scsi 2:0:142:0: SSP: handle(0x002e), sas_addr(0x5000cca26a27af2a), phy(18), device_name(0x5000cca26a27af2b) [ 20.732809] scsi 2:0:142:0: enclosure logical id(0x5000ccab0405dc80), slot(27) [ 20.740114] scsi 2:0:142:0: enclosure level(0x0000), connector name( C1 ) [ 20.746992] scsi 2:0:142:0: serial_number( 2TGPUKSD) [ 20.752563] scsi 2:0:142:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.772572] mpt3sas_cm0: detecting: handle(0x002f), sas_address(0x5000cca26a1bb85a), phy(19) [ 20.781007] mpt3sas_cm0: REPORT_LUNS: handle(0x002f), retries(0) [ 20.787167] mpt3sas_cm0: TEST_UNIT_READY: handle(0x002f), lun(0) [ 20.793821] scsi 2:0:143:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.802212] scsi 2:0:143:0: SSP: handle(0x002f), sas_addr(0x5000cca26a1bb85a), phy(19), device_name(0x5000cca26a1bb85b) [ 20.812984] scsi 2:0:143:0: enclosure logical id(0x5000ccab0405dc80), slot(28) [ 20.820289] scsi 2:0:143:0: enclosure level(0x0000), connector name( C1 ) [ 20.827183] scsi 2:0:143:0: serial_number( 2TGH7LJD) [ 20.832756] scsi 2:0:143:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.853558] mpt3sas_cm0: detecting: handle(0x0030), sas_address(0x5000cca25254e28e), phy(20) [ 20.862000] mpt3sas_cm0: REPORT_LUNS: handle(0x0030), retries(0) [ 20.868140] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0030), lun(0) [ 20.874782] scsi 2:0:144:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.883165] scsi 2:0:144:0: SSP: handle(0x0030), sas_addr(0x5000cca25254e28e), phy(20), device_name(0x5000cca25254e28f) [ 20.893939] scsi 2:0:144:0: enclosure logical id(0x5000ccab0405dc80), slot(29) [ 20.901244] scsi 2:0:144:0: enclosure level(0x0000), connector name( C1 ) [ 20.908139] scsi 2:0:144:0: serial_number( 7SHHP62G) [ 20.913711] scsi 2:0:144:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 20.933559] mpt3sas_cm0: detecting: handle(0x0031), sas_address(0x5000cca26a252dce), phy(21) [ 20.942000] mpt3sas_cm0: REPORT_LUNS: handle(0x0031), retries(0) [ 20.948132] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0031), lun(0) [ 20.955312] scsi 2:0:145:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 20.963705] scsi 2:0:145:0: SSP: handle(0x0031), sas_addr(0x5000cca26a252dce), phy(21), device_name(0x5000cca26a252dcf) [ 20.974479] scsi 2:0:145:0: enclosure logical id(0x5000ccab0405dc80), slot(30) [ 20.981785] scsi 2:0:145:0: enclosure level(0x0000), connector name( C1 ) [ 20.988675] scsi 2:0:145:0: serial_number( 2TGNEVND) [ 20.994251] scsi 2:0:145:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.015153] mpt3sas_cm0: detecting: handle(0x0032), sas_address(0x5000cca25254452a), phy(22) [ 21.023590] mpt3sas_cm0: REPORT_LUNS: handle(0x0032), retries(0) [ 21.029754] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0032), lun(0) [ 21.036390] scsi 2:0:146:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.044800] scsi 2:0:146:0: SSP: handle(0x0032), sas_addr(0x5000cca25254452a), phy(22), device_name(0x5000cca25254452b) [ 21.055571] scsi 2:0:146:0: enclosure logical id(0x5000ccab0405dc80), slot(31) [ 21.062879] scsi 2:0:146:0: enclosure level(0x0000), connector name( C1 ) [ 21.069769] scsi 2:0:146:0: serial_number( 7SHHAR4G) [ 21.075344] scsi 2:0:146:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.098563] mpt3sas_cm0: detecting: handle(0x0033), sas_address(0x5000cca26a23b97a), phy(23) [ 21.106998] mpt3sas_cm0: REPORT_LUNS: handle(0x0033), retries(0) [ 21.113138] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0033), lun(0) [ 21.119776] scsi 2:0:147:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.128171] scsi 2:0:147:0: SSP: handle(0x0033), sas_addr(0x5000cca26a23b97a), phy(23), device_name(0x5000cca26a23b97b) [ 21.138946] scsi 2:0:147:0: enclosure logical id(0x5000ccab0405dc80), slot(32) [ 21.146252] scsi 2:0:147:0: enclosure level(0x0000), connector name( C1 ) [ 21.153145] scsi 2:0:147:0: serial_number( 2TGMN1ZD) [ 21.158718] scsi 2:0:147:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.181564] mpt3sas_cm0: detecting: handle(0x0034), sas_address(0x5000cca25254fbfa), phy(24) [ 21.190008] mpt3sas_cm0: REPORT_LUNS: handle(0x0034), retries(0) [ 21.196145] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0034), lun(0) [ 21.204743] scsi 2:0:148:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.213142] scsi 2:0:148:0: SSP: handle(0x0034), sas_addr(0x5000cca25254fbfa), phy(24), device_name(0x5000cca25254fbfb) [ 21.223912] scsi 2:0:148:0: enclosure logical id(0x5000ccab0405dc80), slot(33) [ 21.231219] scsi 2:0:148:0: enclosure level(0x0000), connector name( C1 ) [ 21.238111] scsi 2:0:148:0: serial_number( 7SHHRWKG) [ 21.243685] scsi 2:0:148:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.263561] mpt3sas_cm0: detecting: handle(0x0035), sas_address(0x5000cca25253e276), phy(25) [ 21.272000] mpt3sas_cm0: REPORT_LUNS: handle(0x0035), retries(0) [ 21.278167] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0035), lun(0) [ 21.286225] scsi 2:0:149:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.294619] scsi 2:0:149:0: SSP: handle(0x0035), sas_addr(0x5000cca25253e276), phy(25), device_name(0x5000cca25253e277) [ 21.305390] scsi 2:0:149:0: enclosure logical id(0x5000ccab0405dc80), slot(34) [ 21.312693] scsi 2:0:149:0: enclosure level(0x0000), connector name( C1 ) [ 21.319587] scsi 2:0:149:0: serial_number( 7SHH44BG) [ 21.325161] scsi 2:0:149:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.347832] mpt3sas_cm0: detecting: handle(0x0036), sas_address(0x5000cca26a235faa), phy(26) [ 21.356268] mpt3sas_cm0: REPORT_LUNS: handle(0x0036), retries(0) [ 21.362530] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0036), lun(0) [ 21.369254] scsi 2:0:150:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.377685] scsi 2:0:150:0: SSP: handle(0x0036), sas_addr(0x5000cca26a235faa), phy(26), device_name(0x5000cca26a235fab) [ 21.388459] scsi 2:0:150:0: enclosure logical id(0x5000ccab0405dc80), slot(35) [ 21.395765] scsi 2:0:150:0: enclosure level(0x0000), connector name( C1 ) [ 21.402659] scsi 2:0:150:0: serial_number( 2TGMG2KD) [ 21.408232] scsi 2:0:150:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.433598] mpt3sas_cm0: detecting: handle(0x0037), sas_address(0x5000cca26a26b086), phy(27) [ 21.442033] mpt3sas_cm0: REPORT_LUNS: handle(0x0037), retries(0) [ 21.448173] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0037), lun(0) [ 21.540693] scsi 2:0:151:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.549075] scsi 2:0:151:0: SSP: handle(0x0037), sas_addr(0x5000cca26a26b086), phy(27), device_name(0x5000cca26a26b087) [ 21.559851] scsi 2:0:151:0: enclosure logical id(0x5000ccab0405dc80), slot(36) [ 21.567157] scsi 2:0:151:0: enclosure level(0x0000), connector name( C1 ) [ 21.574039] scsi 2:0:151:0: serial_number( 2TGP8M1D) [ 21.579614] scsi 2:0:151:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.599569] mpt3sas_cm0: detecting: handle(0x0038), sas_address(0x5000cca26a224eca), phy(28) [ 21.608008] mpt3sas_cm0: REPORT_LUNS: handle(0x0038), retries(0) [ 21.614178] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0038), lun(0) [ 21.620938] scsi 2:0:152:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.629333] scsi 2:0:152:0: SSP: handle(0x0038), sas_addr(0x5000cca26a224eca), phy(28), device_name(0x5000cca26a224ecb) [ 21.640104] scsi 2:0:152:0: enclosure logical id(0x5000ccab0405dc80), slot(37) [ 21.647409] scsi 2:0:152:0: enclosure level(0x0000), connector name( C1 ) [ 21.654300] scsi 2:0:152:0: serial_number( 2TGLVX6D) [ 21.659874] scsi 2:0:152:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.682570] mpt3sas_cm0: detecting: handle(0x0039), sas_address(0x5000cca26a2c58d6), phy(29) [ 21.691008] mpt3sas_cm0: REPORT_LUNS: handle(0x0039), retries(0) [ 21.697153] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0039), lun(0) [ 21.703981] scsi 2:0:153:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.712376] scsi 2:0:153:0: SSP: handle(0x0039), sas_addr(0x5000cca26a2c58d6), phy(29), device_name(0x5000cca26a2c58d7) [ 21.723146] scsi 2:0:153:0: enclosure logical id(0x5000ccab0405dc80), slot(38) [ 21.730454] scsi 2:0:153:0: enclosure level(0x0000), connector name( C1 ) [ 21.737346] scsi 2:0:153:0: serial_number( 2TGTD23D) [ 21.742920] scsi 2:0:153:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.765568] mpt3sas_cm0: detecting: handle(0x003a), sas_address(0x5000cca2525450fe), phy(30) [ 21.774010] mpt3sas_cm0: REPORT_LUNS: handle(0x003a), retries(0) [ 21.780151] mpt3sas_cm0: TEST_UNIT_READY: handle(0x003a), lun(0) [ 21.786915] scsi 2:0:154:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.795305] scsi 2:0:154:0: SSP: handle(0x003a), sas_addr(0x5000cca2525450fe), phy(30), device_name(0x5000cca2525450ff) [ 21.806079] scsi 2:0:154:0: enclosure logical id(0x5000ccab0405dc80), slot(39) [ 21.813384] scsi 2:0:154:0: enclosure level(0x0000), connector name( C1 ) [ 21.820276] scsi 2:0:154:0: serial_number( 7SHHBHKG) [ 21.825850] scsi 2:0:154:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.848567] mpt3sas_cm0: detecting: handle(0x003b), sas_address(0x5000cca266d33c6a), phy(31) [ 21.857009] mpt3sas_cm0: REPORT_LUNS: handle(0x003b), retries(0) [ 21.863177] mpt3sas_cm0: TEST_UNIT_READY: handle(0x003b), lun(0) [ 21.869819] scsi 2:0:155:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.878210] scsi 2:0:155:0: SSP: handle(0x003b), sas_addr(0x5000cca266d33c6a), phy(31), device_name(0x5000cca266d33c6b) [ 21.888983] scsi 2:0:155:0: enclosure logical id(0x5000ccab0405dc80), slot(40) [ 21.896290] scsi 2:0:155:0: enclosure level(0x0000), connector name( C1 ) [ 21.903181] scsi 2:0:155:0: serial_number( 7JKS5AMK) [ 21.908755] scsi 2:0:155:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 21.931583] mpt3sas_cm0: detecting: handle(0x003c), sas_address(0x5000cca26a2435aa), phy(32) [ 21.940017] mpt3sas_cm0: REPORT_LUNS: handle(0x003c), retries(0) [ 21.946148] mpt3sas_cm0: TEST_UNIT_READY: handle(0x003c), lun(0) [ 21.952819] scsi 2:0:156:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 21.961252] scsi 2:0:156:0: SSP: handle(0x003c), sas_addr(0x5000cca26a2435aa), phy(32), device_name(0x5000cca26a2435ab) [ 21.972027] scsi 2:0:156:0: enclosure logical id(0x5000ccab0405dc80), slot(41) [ 21.979334] scsi 2:0:156:0: enclosure level(0x0000), connector name( C1 ) [ 21.986228] scsi 2:0:156:0: serial_number( 2TGMXABD) [ 21.991800] scsi 2:0:156:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.036589] mpt3sas_cm0: detecting: handle(0x003d), sas_address(0x5000cca26807574a), phy(33) [ 22.045022] mpt3sas_cm0: REPORT_LUNS: handle(0x003d), retries(0) [ 22.051212] mpt3sas_cm0: TEST_UNIT_READY: handle(0x003d), lun(0) [ 22.058056] scsi 2:0:157:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.066472] scsi 2:0:157:0: SSP: handle(0x003d), sas_addr(0x5000cca26807574a), phy(33), device_name(0x5000cca26807574b) [ 22.077241] scsi 2:0:157:0: enclosure logical id(0x5000ccab0405dc80), slot(42) [ 22.084547] scsi 2:0:157:0: enclosure level(0x0000), connector name( C1 ) [ 22.091437] scsi 2:0:157:0: serial_number( E4G414UK) [ 22.097012] scsi 2:0:157:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.125599] mpt3sas_cm0: detecting: handle(0x003e), sas_address(0x5000cca26b9a57d2), phy(34) [ 22.134039] mpt3sas_cm0: REPORT_LUNS: handle(0x003e), retries(0) [ 22.140210] mpt3sas_cm0: TEST_UNIT_READY: handle(0x003e), lun(0) [ 22.158651] scsi 2:0:158:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.167072] scsi 2:0:158:0: SSP: handle(0x003e), sas_addr(0x5000cca26b9a57d2), phy(34), device_name(0x5000cca26b9a57d3) [ 22.177842] scsi 2:0:158:0: enclosure logical id(0x5000ccab0405dc80), slot(43) [ 22.185149] scsi 2:0:158:0: enclosure level(0x0000), connector name( C1 ) [ 22.192025] scsi 2:0:158:0: serial_number( 1SJRWBTZ) [ 22.197596] scsi 2:0:158:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.217573] mpt3sas_cm0: detecting: handle(0x003f), sas_address(0x5000cca26b9b2d02), phy(35) [ 22.226009] mpt3sas_cm0: REPORT_LUNS: handle(0x003f), retries(0) [ 22.232151] mpt3sas_cm0: TEST_UNIT_READY: handle(0x003f), lun(0) [ 22.238802] scsi 2:0:159:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.247191] scsi 2:0:159:0: SSP: handle(0x003f), sas_addr(0x5000cca26b9b2d02), phy(35), device_name(0x5000cca26b9b2d03) [ 22.257967] scsi 2:0:159:0: enclosure logical id(0x5000ccab0405dc80), slot(44) [ 22.265271] scsi 2:0:159:0: enclosure level(0x0000), connector name( C1 ) [ 22.272163] scsi 2:0:159:0: serial_number( 1SJSAKXZ) [ 22.277739] scsi 2:0:159:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.297588] mpt3sas_cm0: detecting: handle(0x0040), sas_address(0x5000cca26a2440e2), phy(36) [ 22.306031] mpt3sas_cm0: REPORT_LUNS: handle(0x0040), retries(0) [ 22.312168] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0040), lun(0) [ 22.318822] scsi 2:0:160:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.327209] scsi 2:0:160:0: SSP: handle(0x0040), sas_addr(0x5000cca26a2440e2), phy(36), device_name(0x5000cca26a2440e3) [ 22.337985] scsi 2:0:160:0: enclosure logical id(0x5000ccab0405dc80), slot(45) [ 22.345290] scsi 2:0:160:0: enclosure level(0x0000), connector name( C1 ) [ 22.352183] scsi 2:0:160:0: serial_number( 2TGMY2JD) [ 22.357756] scsi 2:0:160:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.377574] mpt3sas_cm0: detecting: handle(0x0041), sas_address(0x5000cca25254dfee), phy(37) [ 22.386014] mpt3sas_cm0: REPORT_LUNS: handle(0x0041), retries(0) [ 22.392159] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0041), lun(0) [ 22.401297] scsi 2:0:161:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.409697] scsi 2:0:161:0: SSP: handle(0x0041), sas_addr(0x5000cca25254dfee), phy(37), device_name(0x5000cca25254dfef) [ 22.420465] scsi 2:0:161:0: enclosure logical id(0x5000ccab0405dc80), slot(46) [ 22.427771] scsi 2:0:161:0: enclosure level(0x0000), connector name( C1 ) [ 22.434662] scsi 2:0:161:0: serial_number( 7SHHP0NG) [ 22.440239] scsi 2:0:161:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.460571] mpt3sas_cm0: detecting: handle(0x0042), sas_address(0x5000cca3c619496a), phy(38) [ 22.469004] mpt3sas_cm0: REPORT_LUNS: handle(0x0042), retries(0) [ 22.475155] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0042), lun(0) [ 22.481967] scsi 2:0:162:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.490383] scsi 2:0:162:0: SSP: handle(0x0042), sas_addr(0x5000cca3c619496a), phy(38), device_name(0x5000cca3c619496b) [ 22.501152] scsi 2:0:162:0: enclosure logical id(0x5000ccab0405dc80), slot(47) [ 22.508458] scsi 2:0:162:0: enclosure level(0x0000), connector name( C1 ) [ 22.515349] scsi 2:0:162:0: serial_number( 7G0EX3GK) [ 22.520923] scsi 2:0:162:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.543586] mpt3sas_cm0: detecting: handle(0x0043), sas_address(0x5000cca2526b15f2), phy(39) [ 22.552021] mpt3sas_cm0: REPORT_LUNS: handle(0x0043), retries(0) [ 22.558161] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0043), lun(0) [ 22.564815] scsi 2:0:163:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.573203] scsi 2:0:163:0: SSP: handle(0x0043), sas_addr(0x5000cca2526b15f2), phy(39), device_name(0x5000cca2526b15f3) [ 22.583978] scsi 2:0:163:0: enclosure logical id(0x5000ccab0405dc80), slot(48) [ 22.591287] scsi 2:0:163:0: enclosure level(0x0000), connector name( C1 ) [ 22.598182] scsi 2:0:163:0: serial_number( 7SHWWPJU) [ 22.603761] scsi 2:0:163:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.623575] mpt3sas_cm0: detecting: handle(0x0044), sas_address(0x5000cca26b9be876), phy(40) [ 22.632015] mpt3sas_cm0: REPORT_LUNS: handle(0x0044), retries(0) [ 22.638148] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0044), lun(0) [ 22.644798] scsi 2:0:164:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.653191] scsi 2:0:164:0: SSP: handle(0x0044), sas_addr(0x5000cca26b9be876), phy(40), device_name(0x5000cca26b9be877) [ 22.663964] scsi 2:0:164:0: enclosure logical id(0x5000ccab0405dc80), slot(49) [ 22.671271] scsi 2:0:164:0: enclosure level(0x0000), connector name( C1 ) [ 22.678162] scsi 2:0:164:0: serial_number( 1SJSS1XZ) [ 22.683736] scsi 2:0:164:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.703574] mpt3sas_cm0: detecting: handle(0x0045), sas_address(0x5000cca26a259c0e), phy(41) [ 22.712008] mpt3sas_cm0: REPORT_LUNS: handle(0x0045), retries(0) [ 22.718140] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0045), lun(0) [ 22.724777] scsi 2:0:165:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.733164] scsi 2:0:165:0: SSP: handle(0x0045), sas_addr(0x5000cca26a259c0e), phy(41), device_name(0x5000cca26a259c0f) [ 22.743941] scsi 2:0:165:0: enclosure logical id(0x5000ccab0405dc80), slot(50) [ 22.751246] scsi 2:0:165:0: enclosure level(0x0000), connector name( C1 ) [ 22.758137] scsi 2:0:165:0: serial_number( 2TGNP68D) [ 22.763711] scsi 2:0:165:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.783576] mpt3sas_cm0: detecting: handle(0x0046), sas_address(0x5000cca266d7197e), phy(42) [ 22.792011] mpt3sas_cm0: REPORT_LUNS: handle(0x0046), retries(0) [ 22.798151] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0046), lun(0) [ 22.804813] scsi 2:0:166:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.813207] scsi 2:0:166:0: SSP: handle(0x0046), sas_addr(0x5000cca266d7197e), phy(42), device_name(0x5000cca266d7197f) [ 22.823977] scsi 2:0:166:0: enclosure logical id(0x5000ccab0405dc80), slot(51) [ 22.831282] scsi 2:0:166:0: enclosure level(0x0000), connector name( C1 ) [ 22.838175] scsi 2:0:166:0: serial_number( 7JKU96LK) [ 22.843749] scsi 2:0:166:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.863578] mpt3sas_cm0: detecting: handle(0x0047), sas_address(0x5000cca26a2223ae), phy(43) [ 22.872011] mpt3sas_cm0: REPORT_LUNS: handle(0x0047), retries(0) [ 22.878169] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0047), lun(0) [ 22.884821] scsi 2:0:167:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.893214] scsi 2:0:167:0: SSP: handle(0x0047), sas_addr(0x5000cca26a2223ae), phy(43), device_name(0x5000cca26a2223af) [ 22.903986] scsi 2:0:167:0: enclosure logical id(0x5000ccab0405dc80), slot(52) [ 22.911292] scsi 2:0:167:0: enclosure level(0x0000), connector name( C1 ) [ 22.918186] scsi 2:0:167:0: serial_number( 2TGLT16D) [ 22.923757] scsi 2:0:167:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 22.944169] mpt3sas_cm0: detecting: handle(0x0048), sas_address(0x5000cca266d5de42), phy(44) [ 22.952603] mpt3sas_cm0: REPORT_LUNS: handle(0x0048), retries(0) [ 22.958765] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0048), lun(0) [ 22.965414] scsi 2:0:168:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 22.973804] scsi 2:0:168:0: SSP: handle(0x0048), sas_addr(0x5000cca266d5de42), phy(44), device_name(0x5000cca266d5de43) [ 22.984577] scsi 2:0:168:0: enclosure logical id(0x5000ccab0405dc80), slot(53) [ 22.991883] scsi 2:0:168:0: enclosure level(0x0000), connector name( C1 ) [ 22.998774] scsi 2:0:168:0: serial_number( 7JKTM6TK) [ 23.004348] scsi 2:0:168:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.024576] mpt3sas_cm0: detecting: handle(0x0049), sas_address(0x5000cca26b9d3156), phy(45) [ 23.033011] mpt3sas_cm0: REPORT_LUNS: handle(0x0049), retries(0) [ 23.039147] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0049), lun(0) [ 23.045823] scsi 2:0:169:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.054218] scsi 2:0:169:0: SSP: handle(0x0049), sas_addr(0x5000cca26b9d3156), phy(45), device_name(0x5000cca26b9d3157) [ 23.064988] scsi 2:0:169:0: enclosure logical id(0x5000ccab0405dc80), slot(54) [ 23.072292] scsi 2:0:169:0: enclosure level(0x0000), connector name( C1 ) [ 23.079186] scsi 2:0:169:0: serial_number( 1SJTEYWZ) [ 23.084760] scsi 2:0:169:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.104582] mpt3sas_cm0: detecting: handle(0x004a), sas_address(0x5000cca26a24fce6), phy(46) [ 23.113021] mpt3sas_cm0: REPORT_LUNS: handle(0x004a), retries(0) [ 23.119160] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004a), lun(0) [ 23.125830] scsi 2:0:170:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.134217] scsi 2:0:170:0: SSP: handle(0x004a), sas_addr(0x5000cca26a24fce6), phy(46), device_name(0x5000cca26a24fce7) [ 23.144988] scsi 2:0:170:0: enclosure logical id(0x5000ccab0405dc80), slot(55) [ 23.152294] scsi 2:0:170:0: enclosure level(0x0000), connector name( C1 ) [ 23.159188] scsi 2:0:170:0: serial_number( 2TGNALPD) [ 23.164762] scsi 2:0:170:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.184581] mpt3sas_cm0: detecting: handle(0x004b), sas_address(0x5000cca26a26f9aa), phy(47) [ 23.193017] mpt3sas_cm0: REPORT_LUNS: handle(0x004b), retries(0) [ 23.199156] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004b), lun(0) [ 23.205799] scsi 2:0:171:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.214189] scsi 2:0:171:0: SSP: handle(0x004b), sas_addr(0x5000cca26a26f9aa), phy(47), device_name(0x5000cca26a26f9ab) [ 23.224964] scsi 2:0:171:0: enclosure logical id(0x5000ccab0405dc80), slot(56) [ 23.232270] scsi 2:0:171:0: enclosure level(0x0000), connector name( C1 ) [ 23.239163] scsi 2:0:171:0: serial_number( 2TGPEH1D) [ 23.244736] scsi 2:0:171:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.267582] mpt3sas_cm0: detecting: handle(0x004c), sas_address(0x5000cca26a261b16), phy(48) [ 23.276017] mpt3sas_cm0: REPORT_LUNS: handle(0x004c), retries(0) [ 23.282158] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004c), lun(0) [ 23.288790] scsi 2:0:172:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.297172] scsi 2:0:172:0: SSP: handle(0x004c), sas_addr(0x5000cca26a261b16), phy(48), device_name(0x5000cca26a261b17) [ 23.307948] scsi 2:0:172:0: enclosure logical id(0x5000ccab0405dc80), slot(57) [ 23.315254] scsi 2:0:172:0: enclosure level(0x0000), connector name( C1 ) [ 23.322145] scsi 2:0:172:0: serial_number( 2TGNYNJD) [ 23.327720] scsi 2:0:172:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.347582] mpt3sas_cm0: detecting: handle(0x004d), sas_address(0x5000cca26a23dffe), phy(49) [ 23.356019] mpt3sas_cm0: REPORT_LUNS: handle(0x004d), retries(0) [ 23.362181] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004d), lun(0) [ 23.382007] scsi 2:0:173:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.390381] scsi 2:0:173:0: SSP: handle(0x004d), sas_addr(0x5000cca26a23dffe), phy(49), device_name(0x5000cca26a23dfff) [ 23.401149] scsi 2:0:173:0: enclosure logical id(0x5000ccab0405dc80), slot(58) [ 23.408456] scsi 2:0:173:0: enclosure level(0x0000), connector name( C1 ) [ 23.415333] scsi 2:0:173:0: serial_number( 2TGMRMHD) [ 23.420912] scsi 2:0:173:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.449620] mpt3sas_cm0: detecting: handle(0x004e), sas_address(0x5000cca26a258ede), phy(50) [ 23.458060] mpt3sas_cm0: REPORT_LUNS: handle(0x004e), retries(0) [ 23.471722] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004e), lun(0) [ 23.481340] scsi 2:0:174:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.489733] scsi 2:0:174:0: SSP: handle(0x004e), sas_addr(0x5000cca26a258ede), phy(50), device_name(0x5000cca26a258edf) [ 23.500503] scsi 2:0:174:0: enclosure logical id(0x5000ccab0405dc80), slot(59) [ 23.507810] scsi 2:0:174:0: enclosure level(0x0000), connector name( C1 ) [ 23.514702] scsi 2:0:174:0: serial_number( 2TGNNA1D) [ 23.520275] scsi 2:0:174:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.543058] mpt3sas_cm0: expander_add: handle(0x001a), parent(0x0017), sas_addr(0x5000ccab0405dcfb), phys(68) [ 23.564387] mpt3sas_cm0: detecting: handle(0x004f), sas_address(0x5000cca25254e76e), phy(42) [ 23.572827] mpt3sas_cm0: REPORT_LUNS: handle(0x004f), retries(0) [ 23.578960] mpt3sas_cm0: TEST_UNIT_READY: handle(0x004f), lun(0) [ 23.585661] scsi 2:0:175:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.594061] scsi 2:0:175:0: SSP: handle(0x004f), sas_addr(0x5000cca25254e76e), phy(42), device_name(0x5000cca25254e76f) [ 23.604834] scsi 2:0:175:0: enclosure logical id(0x5000ccab0405dc80), slot(1) [ 23.612053] scsi 2:0:175:0: enclosure level(0x0000), connector name( C1 ) [ 23.618943] scsi 2:0:175:0: serial_number( 7SHHPJ4G) [ 23.624517] scsi 2:0:175:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.644585] mpt3sas_cm0: detecting: handle(0x0050), sas_address(0x5000cca25255997e), phy(43) [ 23.653027] mpt3sas_cm0: REPORT_LUNS: handle(0x0050), retries(0) [ 23.659198] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0050), lun(0) [ 23.666106] scsi 2:0:176:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.685211] scsi 2:0:176:0: SSP: handle(0x0050), sas_addr(0x5000cca25255997e), phy(43), device_name(0x5000cca25255997f) [ 23.695979] scsi 2:0:176:0: enclosure logical id(0x5000ccab0405dc80), slot(3) [ 23.703199] scsi 2:0:176:0: enclosure level(0x0000), connector name( C1 ) [ 23.710089] scsi 2:0:176:0: serial_number( 7SHJ2BSG) [ 23.715664] scsi 2:0:176:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.736485] mpt3sas_cm0: detecting: handle(0x0051), sas_address(0x5000cca25255a06a), phy(44) [ 23.744926] mpt3sas_cm0: REPORT_LUNS: handle(0x0051), retries(0) [ 23.751091] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0051), lun(0) [ 23.757757] scsi 2:0:177:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.766145] scsi 2:0:177:0: SSP: handle(0x0051), sas_addr(0x5000cca25255a06a), phy(44), device_name(0x5000cca25255a06b) [ 23.776917] scsi 2:0:177:0: enclosure logical id(0x5000ccab0405dc80), slot(4) [ 23.784138] scsi 2:0:177:0: enclosure level(0x0000), connector name( C1 ) [ 23.791029] scsi 2:0:177:0: serial_number( 7SHJ2V1G) [ 23.796602] scsi 2:0:177:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.816628] mpt3sas_cm0: detecting: handle(0x0052), sas_address(0x5000cca252561fbe), phy(45) [ 23.825065] mpt3sas_cm0: REPORT_LUNS: handle(0x0052), retries(0) [ 23.831198] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0052), lun(0) [ 23.837880] scsi 2:0:178:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.846271] scsi 2:0:178:0: SSP: handle(0x0052), sas_addr(0x5000cca252561fbe), phy(45), device_name(0x5000cca252561fbf) [ 23.857041] scsi 2:0:178:0: enclosure logical id(0x5000ccab0405dc80), slot(5) [ 23.864260] scsi 2:0:178:0: enclosure level(0x0000), connector name( C1 ) [ 23.871154] scsi 2:0:178:0: serial_number( 7SHJB9XG) [ 23.876726] scsi 2:0:178:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.896591] mpt3sas_cm0: detecting: handle(0x0053), sas_address(0x5000cca25254f76e), phy(46) [ 23.905024] mpt3sas_cm0: REPORT_LUNS: handle(0x0053), retries(0) [ 23.911157] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0053), lun(0) [ 23.917820] scsi 2:0:179:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 23.926208] scsi 2:0:179:0: SSP: handle(0x0053), sas_addr(0x5000cca25254f76e), phy(46), device_name(0x5000cca25254f76f) [ 23.936981] scsi 2:0:179:0: enclosure logical id(0x5000ccab0405dc80), slot(6) [ 23.944201] scsi 2:0:179:0: enclosure level(0x0000), connector name( C1 ) [ 23.951095] scsi 2:0:179:0: serial_number( 7SHHRL5G) [ 23.956667] scsi 2:0:179:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 23.976591] mpt3sas_cm0: detecting: handle(0x0054), sas_address(0x5000cca252544e66), phy(47) [ 23.985026] mpt3sas_cm0: REPORT_LUNS: handle(0x0054), retries(0) [ 23.991165] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0054), lun(0) [ 24.000092] scsi 2:0:180:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.008493] scsi 2:0:180:0: SSP: handle(0x0054), sas_addr(0x5000cca252544e66), phy(47), device_name(0x5000cca252544e67) [ 24.019263] scsi 2:0:180:0: enclosure logical id(0x5000ccab0405dc80), slot(7) [ 24.026484] scsi 2:0:180:0: enclosure level(0x0000), connector name( C1 ) [ 24.033373] scsi 2:0:180:0: serial_number( 7SHHBA6G) [ 24.038949] scsi 2:0:180:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.061587] mpt3sas_cm0: detecting: handle(0x0055), sas_address(0x5000cca252562c2a), phy(48) [ 24.070020] mpt3sas_cm0: REPORT_LUNS: handle(0x0055), retries(0) [ 24.076152] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0055), lun(0) [ 24.082888] scsi 2:0:181:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.091280] scsi 2:0:181:0: SSP: handle(0x0055), sas_addr(0x5000cca252562c2a), phy(48), device_name(0x5000cca252562c2b) [ 24.102055] scsi 2:0:181:0: enclosure logical id(0x5000ccab0405dc80), slot(8) [ 24.109276] scsi 2:0:181:0: enclosure level(0x0000), connector name( C1 ) [ 24.116167] scsi 2:0:181:0: serial_number( 7SHJD4KG) [ 24.121742] scsi 2:0:181:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.144598] mpt3sas_cm0: detecting: handle(0x0056), sas_address(0x5000cca25255d15e), phy(49) [ 24.153038] mpt3sas_cm0: REPORT_LUNS: handle(0x0056), retries(0) [ 24.159182] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0056), lun(0) [ 24.165848] scsi 2:0:182:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.174248] scsi 2:0:182:0: SSP: handle(0x0056), sas_addr(0x5000cca25255d15e), phy(49), device_name(0x5000cca25255d15f) [ 24.185023] scsi 2:0:182:0: enclosure logical id(0x5000ccab0405dc80), slot(9) [ 24.192241] scsi 2:0:182:0: enclosure level(0x0000), connector name( C1 ) [ 24.199132] scsi 2:0:182:0: serial_number( 7SHJ633G) [ 24.204707] scsi 2:0:182:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.227603] mpt3sas_cm0: detecting: handle(0x0057), sas_address(0x5000cca25253e25e), phy(50) [ 24.236039] mpt3sas_cm0: REPORT_LUNS: handle(0x0057), retries(0) [ 24.242208] mpt3sas_cm0: TEST_UNIT_READY: handle(0x0057), lun(0) [ 24.249032] scsi 2:0:183:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.257431] scsi 2:0:183:0: SSP: handle(0x0057), sas_addr(0x5000cca25253e25e), phy(50), device_name(0x5000cca25253e25f) [ 24.268204] scsi 2:0:183:0: enclosure logical id(0x5000ccab0405dc80), slot(10) [ 24.275510] scsi 2:0:183:0: enclosure level(0x0000), connector name( C1 ) [ 24.282402] scsi 2:0:183:0: serial_number( 7SHH445G) [ 24.287975] scsi 2:0:183:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.319137] mpt3sas_cm0: expander_add: handle(0x0099), parent(0x000a), sas_addr(0x5000ccab0405dcbd), phys(49) [ 24.340907] mpt3sas_cm0: detecting: handle(0x009d), sas_address(0x5000ccab0405dcbc), phy(48) [ 24.349348] mpt3sas_cm0: REPORT_LUNS: handle(0x009d), retries(0) [ 24.355783] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009d), lun(0) [ 24.362811] scsi 2:0:184:0: Enclosure HGST H4060-J 2033 PQ: 0 ANSI: 6 [ 24.372615] scsi 2:0:184:0: set ignore_delay_remove for handle(0x009d) [ 24.379138] scsi 2:0:184:0: SES: handle(0x009d), sas_addr(0x5000ccab0405dcbc), phy(48), device_name(0x0000000000000000) [ 24.389911] scsi 2:0:184:0: enclosure logical id(0x5000ccab0405dc80), slot(60) [ 24.397216] scsi 2:0:184:0: enclosure level(0x0000), connector name( C0 ) [ 24.404112] scsi 2:0:184:0: serial_number(USWSJ03918EZ0068 ) [ 24.410038] scsi 2:0:184:0: qdepth(1), tagged(0), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.434870] mpt3sas_cm0: expander_add: handle(0x009b), parent(0x0099), sas_addr(0x5000ccab0405dcbf), phys(68) [ 24.457637] mpt3sas_cm0: detecting: handle(0x009e), sas_address(0x5000cca252561065), phy(0) [ 24.465991] mpt3sas_cm0: REPORT_LUNS: handle(0x009e), retries(0) [ 24.472119] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009e), lun(0) [ 24.480376] scsi 2:0:185:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.488770] scsi 2:0:185:0: SSP: handle(0x009e), sas_addr(0x5000cca252561065), phy(0), device_name(0x5000cca252561067) [ 24.499456] scsi 2:0:185:0: enclosure logical id(0x5000ccab0405dc80), slot(0) [ 24.506677] scsi 2:0:185:0: enclosure level(0x0000), connector name( C0 ) [ 24.513569] scsi 2:0:185:0: serial_number( 7SHJA97G) [ 24.519142] scsi 2:0:185:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.541598] mpt3sas_cm0: detecting: handle(0x009f), sas_address(0x5000cca25253f345), phy(1) [ 24.549950] mpt3sas_cm0: REPORT_LUNS: handle(0x009f), retries(0) [ 24.556086] mpt3sas_cm0: TEST_UNIT_READY: handle(0x009f), lun(0) [ 24.562733] scsi 2:0:186:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.571122] scsi 2:0:186:0: SSP: handle(0x009f), sas_addr(0x5000cca25253f345), phy(1), device_name(0x5000cca25253f347) [ 24.581808] scsi 2:0:186:0: enclosure logical id(0x5000ccab0405dc80), slot(2) [ 24.589026] scsi 2:0:186:0: enclosure level(0x0000), connector name( C0 ) [ 24.595922] scsi 2:0:186:0: serial_number( 7SHH582G) [ 24.601500] scsi 2:0:186:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.621599] mpt3sas_cm0: detecting: handle(0x00a0), sas_address(0x5000cca252543b51), phy(2) [ 24.629945] mpt3sas_cm0: REPORT_LUNS: handle(0x00a0), retries(0) [ 24.636100] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a0), lun(0) [ 24.642853] scsi 2:0:187:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.651251] scsi 2:0:187:0: SSP: handle(0x00a0), sas_addr(0x5000cca252543b51), phy(2), device_name(0x5000cca252543b53) [ 24.661938] scsi 2:0:187:0: enclosure logical id(0x5000ccab0405dc80), slot(11) [ 24.669246] scsi 2:0:187:0: enclosure level(0x0000), connector name( C0 ) [ 24.676137] scsi 2:0:187:0: serial_number( 7SHHA1UG) [ 24.681712] scsi 2:0:187:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.704601] mpt3sas_cm0: detecting: handle(0x00a1), sas_address(0x5000cca252559f4d), phy(3) [ 24.712958] mpt3sas_cm0: REPORT_LUNS: handle(0x00a1), retries(0) [ 24.719097] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a1), lun(0) [ 24.725745] scsi 2:0:188:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.734136] scsi 2:0:188:0: SSP: handle(0x00a1), sas_addr(0x5000cca252559f4d), phy(3), device_name(0x5000cca252559f4f) [ 24.744819] scsi 2:0:188:0: enclosure logical id(0x5000ccab0405dc80), slot(12) [ 24.752124] scsi 2:0:188:0: enclosure level(0x0000), connector name( C0 ) [ 24.759016] scsi 2:0:188:0: serial_number( 7SHJ2SSG) [ 24.764589] scsi 2:0:188:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.784602] mpt3sas_cm0: detecting: handle(0x00a2), sas_address(0x5000cca252544e4d), phy(4) [ 24.792951] mpt3sas_cm0: REPORT_LUNS: handle(0x00a2), retries(0) [ 24.799120] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a2), lun(0) [ 24.805786] scsi 2:0:189:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.814191] scsi 2:0:189:0: SSP: handle(0x00a2), sas_addr(0x5000cca252544e4d), phy(4), device_name(0x5000cca252544e4f) [ 24.824882] scsi 2:0:189:0: enclosure logical id(0x5000ccab0405dc80), slot(13) [ 24.832186] scsi 2:0:189:0: enclosure level(0x0000), connector name( C0 ) [ 24.839080] scsi 2:0:189:0: serial_number( 7SHHBA0G) [ 24.844653] scsi 2:0:189:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.864601] mpt3sas_cm0: detecting: handle(0x00a3), sas_address(0x5000cca25253e40d), phy(5) [ 24.872949] mpt3sas_cm0: REPORT_LUNS: handle(0x00a3), retries(0) [ 24.879140] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a3), lun(0) [ 24.886012] scsi 2:0:190:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.894427] scsi 2:0:190:0: SSP: handle(0x00a3), sas_addr(0x5000cca25253e40d), phy(5), device_name(0x5000cca25253e40f) [ 24.905118] scsi 2:0:190:0: enclosure logical id(0x5000ccab0405dc80), slot(14) [ 24.912422] scsi 2:0:190:0: enclosure level(0x0000), connector name( C0 ) [ 24.919314] scsi 2:0:190:0: serial_number( 7SHH47NG) [ 24.924887] scsi 2:0:190:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 24.947615] mpt3sas_cm0: detecting: handle(0x00a4), sas_address(0x5000cca26b9adc29), phy(6) [ 24.955969] mpt3sas_cm0: REPORT_LUNS: handle(0x00a4), retries(0) [ 24.962157] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a4), lun(0) [ 24.969034] scsi 2:0:191:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 24.977458] scsi 2:0:191:0: SSP: handle(0x00a4), sas_addr(0x5000cca26b9adc29), phy(6), device_name(0x5000cca26b9adc2b) [ 24.988142] scsi 2:0:191:0: enclosure logical id(0x5000ccab0405dc80), slot(15) [ 24.995449] scsi 2:0:191:0: enclosure level(0x0000), connector name( C0 ) [ 25.002342] scsi 2:0:191:0: serial_number( 1SJS560Z) [ 25.007914] scsi 2:0:191:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.038629] mpt3sas_cm0: detecting: handle(0x00a5), sas_address(0x5000cca252550a7d), phy(7) [ 25.046978] mpt3sas_cm0: REPORT_LUNS: handle(0x00a5), retries(0) [ 25.053146] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a5), lun(0) [ 25.059815] scsi 2:0:192:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.068204] scsi 2:0:192:0: SSP: handle(0x00a5), sas_addr(0x5000cca252550a7d), phy(7), device_name(0x5000cca252550a7f) [ 25.078892] scsi 2:0:192:0: enclosure logical id(0x5000ccab0405dc80), slot(16) [ 25.086197] scsi 2:0:192:0: enclosure level(0x0000), connector name( C0 ) [ 25.093092] scsi 2:0:192:0: serial_number( 7SHHSVJG) [ 25.098671] scsi 2:0:192:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.123937] mpt3sas_cm0: detecting: handle(0x00a6), sas_address(0x5000cca25253fc91), phy(8) [ 25.132295] mpt3sas_cm0: REPORT_LUNS: handle(0x00a6), retries(0) [ 25.138464] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a6), lun(0) [ 25.145106] scsi 2:0:193:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.153502] scsi 2:0:193:0: SSP: handle(0x00a6), sas_addr(0x5000cca25253fc91), phy(8), device_name(0x5000cca25253fc93) [ 25.164188] scsi 2:0:193:0: enclosure logical id(0x5000ccab0405dc80), slot(17) [ 25.171493] scsi 2:0:193:0: enclosure level(0x0000), connector name( C0 ) [ 25.178386] scsi 2:0:193:0: serial_number( 7SHH5W8G) [ 25.183961] scsi 2:0:193:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.206603] mpt3sas_cm0: detecting: handle(0x00a7), sas_address(0x5000cca26b9d2289), phy(9) [ 25.214965] mpt3sas_cm0: REPORT_LUNS: handle(0x00a7), retries(0) [ 25.221207] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a7), lun(0) [ 25.228128] scsi 2:0:194:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.240846] scsi 2:0:194:0: SSP: handle(0x00a7), sas_addr(0x5000cca26b9d2289), phy(9), device_name(0x5000cca26b9d228b) [ 25.251530] scsi 2:0:194:0: enclosure logical id(0x5000ccab0405dc80), slot(18) [ 25.258838] scsi 2:0:194:0: enclosure level(0x0000), connector name( C0 ) [ 25.265731] scsi 2:0:194:0: serial_number( 1SJTDZAZ) [ 25.271303] scsi 2:0:194:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.300740] mpt3sas_cm0: detecting: handle(0x00a8), sas_address(0x5000cca2525448f5), phy(10) [ 25.309179] mpt3sas_cm0: REPORT_LUNS: handle(0x00a8), retries(0) [ 25.315342] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a8), lun(0) [ 25.322752] scsi 2:0:195:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.331266] scsi 2:0:195:0: SSP: handle(0x00a8), sas_addr(0x5000cca2525448f5), phy(10), device_name(0x5000cca2525448f7) [ 25.342037] scsi 2:0:195:0: enclosure logical id(0x5000ccab0405dc80), slot(19) [ 25.349342] scsi 2:0:195:0: enclosure level(0x0000), connector name( C0 ) [ 25.356235] scsi 2:0:195:0: serial_number( 7SHHAYZG) [ 25.361808] scsi 2:0:195:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.399606] mpt3sas_cm0: detecting: handle(0x00a9), sas_address(0x5000cca26a234e2d), phy(11) [ 25.408047] mpt3sas_cm0: REPORT_LUNS: handle(0x00a9), retries(0) [ 25.414511] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00a9), lun(0) [ 25.421268] scsi 2:0:196:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.429663] scsi 2:0:196:0: SSP: handle(0x00a9), sas_addr(0x5000cca26a234e2d), phy(11), device_name(0x5000cca26a234e2f) [ 25.440439] scsi 2:0:196:0: enclosure logical id(0x5000ccab0405dc80), slot(20) [ 25.447744] scsi 2:0:196:0: enclosure level(0x0000), connector name( C0 ) [ 25.454638] scsi 2:0:196:0: serial_number( 2TGMDXGD) [ 25.460211] scsi 2:0:196:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.482606] mpt3sas_cm0: detecting: handle(0x00aa), sas_address(0x5000cca26a220395), phy(12) [ 25.491041] mpt3sas_cm0: REPORT_LUNS: handle(0x00aa), retries(0) [ 25.497183] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00aa), lun(0) [ 25.503859] scsi 2:0:197:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.512305] scsi 2:0:197:0: SSP: handle(0x00aa), sas_addr(0x5000cca26a220395), phy(12), device_name(0x5000cca26a220397) [ 25.523075] scsi 2:0:197:0: enclosure logical id(0x5000ccab0405dc80), slot(21) [ 25.530380] scsi 2:0:197:0: enclosure level(0x0000), connector name( C0 ) [ 25.537273] scsi 2:0:197:0: serial_number( 2TGLPWYD) [ 25.542848] scsi 2:0:197:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.563606] mpt3sas_cm0: detecting: handle(0x00ab), sas_address(0x5000cca26a228cb1), phy(13) [ 25.572045] mpt3sas_cm0: REPORT_LUNS: handle(0x00ab), retries(0) [ 25.578199] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ab), lun(0) [ 25.592889] scsi 2:0:198:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.601269] scsi 2:0:198:0: SSP: handle(0x00ab), sas_addr(0x5000cca26a228cb1), phy(13), device_name(0x5000cca26a228cb3) [ 25.612039] scsi 2:0:198:0: enclosure logical id(0x5000ccab0405dc80), slot(22) [ 25.619344] scsi 2:0:198:0: enclosure level(0x0000), connector name( C0 ) [ 25.626222] scsi 2:0:198:0: serial_number( 2TGM010D) [ 25.631800] scsi 2:0:198:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.651606] mpt3sas_cm0: detecting: handle(0x00ac), sas_address(0x5000cca26a2223ed), phy(14) [ 25.660048] mpt3sas_cm0: REPORT_LUNS: handle(0x00ac), retries(0) [ 25.666214] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ac), lun(0) [ 25.681102] scsi 2:0:199:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.689498] scsi 2:0:199:0: SSP: handle(0x00ac), sas_addr(0x5000cca26a2223ed), phy(14), device_name(0x5000cca26a2223ef) [ 25.700272] scsi 2:0:199:0: enclosure logical id(0x5000ccab0405dc80), slot(23) [ 25.707579] scsi 2:0:199:0: enclosure level(0x0000), connector name( C0 ) [ 25.714459] scsi 2:0:199:0: serial_number( 2TGLT1RD) [ 25.720036] scsi 2:0:199:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.742607] mpt3sas_cm0: detecting: handle(0x00ad), sas_address(0x5000cca26a288e35), phy(15) [ 25.751047] mpt3sas_cm0: REPORT_LUNS: handle(0x00ad), retries(0) [ 25.757204] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ad), lun(0) [ 25.763957] scsi 2:0:200:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.772346] scsi 2:0:200:0: SSP: handle(0x00ad), sas_addr(0x5000cca26a288e35), phy(15), device_name(0x5000cca26a288e37) [ 25.783118] scsi 2:0:200:0: enclosure logical id(0x5000ccab0405dc80), slot(24) [ 25.790423] scsi 2:0:200:0: enclosure level(0x0000), connector name( C0 ) [ 25.797317] scsi 2:0:200:0: serial_number( 2TGR9E7D) [ 25.802887] scsi 2:0:200:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.825625] mpt3sas_cm0: detecting: handle(0x00ae), sas_address(0x5000cca26a24abfd), phy(16) [ 25.834066] mpt3sas_cm0: REPORT_LUNS: handle(0x00ae), retries(0) [ 25.840206] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ae), lun(0) [ 25.846870] scsi 2:0:201:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.855259] scsi 2:0:201:0: SSP: handle(0x00ae), sas_addr(0x5000cca26a24abfd), phy(16), device_name(0x5000cca26a24abff) [ 25.866031] scsi 2:0:201:0: enclosure logical id(0x5000ccab0405dc80), slot(25) [ 25.873338] scsi 2:0:201:0: enclosure level(0x0000), connector name( C0 ) [ 25.880228] scsi 2:0:201:0: serial_number( 2TGN56ND) [ 25.885802] scsi 2:0:201:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.905609] mpt3sas_cm0: detecting: handle(0x00af), sas_address(0x5000cca26a26cb05), phy(17) [ 25.914050] mpt3sas_cm0: REPORT_LUNS: handle(0x00af), retries(0) [ 25.920239] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00af), lun(0) [ 25.927178] scsi 2:0:202:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 25.935594] scsi 2:0:202:0: SSP: handle(0x00af), sas_addr(0x5000cca26a26cb05), phy(17), device_name(0x5000cca26a26cb07) [ 25.946362] scsi 2:0:202:0: enclosure logical id(0x5000ccab0405dc80), slot(26) [ 25.953668] scsi 2:0:202:0: enclosure level(0x0000), connector name( C0 ) [ 25.960561] scsi 2:0:202:0: serial_number( 2TGPABSD) [ 25.966135] scsi 2:0:202:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 25.988638] mpt3sas_cm0: detecting: handle(0x00b0), sas_address(0x5000cca26a27af29), phy(18) [ 25.997079] mpt3sas_cm0: REPORT_LUNS: handle(0x00b0), retries(0) [ 26.003246] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b0), lun(0) [ 26.010169] scsi 2:0:203:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.018592] scsi 2:0:203:0: SSP: handle(0x00b0), sas_addr(0x5000cca26a27af29), phy(18), device_name(0x5000cca26a27af2b) [ 26.029362] scsi 2:0:203:0: enclosure logical id(0x5000ccab0405dc80), slot(27) [ 26.036669] scsi 2:0:203:0: enclosure level(0x0000), connector name( C0 ) [ 26.043563] scsi 2:0:203:0: serial_number( 2TGPUKSD) [ 26.049135] scsi 2:0:203:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.071660] mpt3sas_cm0: detecting: handle(0x00b1), sas_address(0x5000cca26a1bb859), phy(19) [ 26.080095] mpt3sas_cm0: REPORT_LUNS: handle(0x00b1), retries(0) [ 26.086261] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b1), lun(0) [ 26.092904] scsi 2:0:204:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.101297] scsi 2:0:204:0: SSP: handle(0x00b1), sas_addr(0x5000cca26a1bb859), phy(19), device_name(0x5000cca26a1bb85b) [ 26.112069] scsi 2:0:204:0: enclosure logical id(0x5000ccab0405dc80), slot(28) [ 26.119375] scsi 2:0:204:0: enclosure level(0x0000), connector name( C0 ) [ 26.126266] scsi 2:0:204:0: serial_number( 2TGH7LJD) [ 26.131839] scsi 2:0:204:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.154611] mpt3sas_cm0: detecting: handle(0x00b2), sas_address(0x5000cca25254e28d), phy(20) [ 26.163052] mpt3sas_cm0: REPORT_LUNS: handle(0x00b2), retries(0) [ 26.169222] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b2), lun(0) [ 26.175880] scsi 2:0:205:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.184269] scsi 2:0:205:0: SSP: handle(0x00b2), sas_addr(0x5000cca25254e28d), phy(20), device_name(0x5000cca25254e28f) [ 26.195045] scsi 2:0:205:0: enclosure logical id(0x5000ccab0405dc80), slot(29) [ 26.202349] scsi 2:0:205:0: enclosure level(0x0000), connector name( C0 ) [ 26.209242] scsi 2:0:205:0: serial_number( 7SHHP62G) [ 26.214814] scsi 2:0:205:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.234610] mpt3sas_cm0: detecting: handle(0x00b3), sas_address(0x5000cca26a252dcd), phy(21) [ 26.243044] mpt3sas_cm0: REPORT_LUNS: handle(0x00b3), retries(0) [ 26.249174] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b3), lun(0) [ 26.255826] scsi 2:0:206:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.264211] scsi 2:0:206:0: SSP: handle(0x00b3), sas_addr(0x5000cca26a252dcd), phy(21), device_name(0x5000cca26a252dcf) [ 26.274984] scsi 2:0:206:0: enclosure logical id(0x5000ccab0405dc80), slot(30) [ 26.282292] scsi 2:0:206:0: enclosure level(0x0000), connector name( C0 ) [ 26.289181] scsi 2:0:206:0: serial_number( 2TGNEVND) [ 26.294756] scsi 2:0:206:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.315203] mpt3sas_cm0: detecting: handle(0x00b4), sas_address(0x5000cca252544529), phy(22) [ 26.323646] mpt3sas_cm0: REPORT_LUNS: handle(0x00b4), retries(0) [ 26.329811] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b4), lun(0) [ 26.336488] scsi 2:0:207:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.344883] scsi 2:0:207:0: SSP: handle(0x00b4), sas_addr(0x5000cca252544529), phy(22), device_name(0x5000cca25254452b) [ 26.355654] scsi 2:0:207:0: enclosure logical id(0x5000ccab0405dc80), slot(31) [ 26.362959] scsi 2:0:207:0: enclosure level(0x0000), connector name( C0 ) [ 26.369853] scsi 2:0:207:0: serial_number( 7SHHAR4G) [ 26.375425] scsi 2:0:207:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.395614] mpt3sas_cm0: detecting: handle(0x00b5), sas_address(0x5000cca26a23b979), phy(23) [ 26.404054] mpt3sas_cm0: REPORT_LUNS: handle(0x00b5), retries(0) [ 26.410194] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b5), lun(0) [ 26.416857] scsi 2:0:208:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.425248] scsi 2:0:208:0: SSP: handle(0x00b5), sas_addr(0x5000cca26a23b979), phy(23), device_name(0x5000cca26a23b97b) [ 26.436019] scsi 2:0:208:0: enclosure logical id(0x5000ccab0405dc80), slot(32) [ 26.443326] scsi 2:0:208:0: enclosure level(0x0000), connector name( C0 ) [ 26.450217] scsi 2:0:208:0: serial_number( 2TGMN1ZD) [ 26.455792] scsi 2:0:208:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.475613] mpt3sas_cm0: detecting: handle(0x00b6), sas_address(0x5000cca25254fbf9), phy(24) [ 26.484056] mpt3sas_cm0: REPORT_LUNS: handle(0x00b6), retries(0) [ 26.490224] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b6), lun(0) [ 26.496891] scsi 2:0:209:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.505281] scsi 2:0:209:0: SSP: handle(0x00b6), sas_addr(0x5000cca25254fbf9), phy(24), device_name(0x5000cca25254fbfb) [ 26.516055] scsi 2:0:209:0: enclosure logical id(0x5000ccab0405dc80), slot(33) [ 26.523362] scsi 2:0:209:0: enclosure level(0x0000), connector name( C0 ) [ 26.530254] scsi 2:0:209:0: serial_number( 7SHHRWKG) [ 26.535827] scsi 2:0:209:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.558612] mpt3sas_cm0: detecting: handle(0x00b7), sas_address(0x5000cca25253e275), phy(25) [ 26.567049] mpt3sas_cm0: REPORT_LUNS: handle(0x00b7), retries(0) [ 26.573196] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b7), lun(0) [ 26.579951] scsi 2:0:210:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.588346] scsi 2:0:210:0: SSP: handle(0x00b7), sas_addr(0x5000cca25253e275), phy(25), device_name(0x5000cca25253e277) [ 26.599117] scsi 2:0:210:0: enclosure logical id(0x5000ccab0405dc80), slot(34) [ 26.606424] scsi 2:0:210:0: enclosure level(0x0000), connector name( C0 ) [ 26.613315] scsi 2:0:210:0: serial_number( 7SHH44BG) [ 26.618889] scsi 2:0:210:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.641644] mpt3sas_cm0: detecting: handle(0x00b8), sas_address(0x5000cca26a235fa9), phy(26) [ 26.650082] mpt3sas_cm0: REPORT_LUNS: handle(0x00b8), retries(0) [ 26.656247] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b8), lun(0) [ 26.672523] scsi 2:0:211:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.680909] scsi 2:0:211:0: SSP: handle(0x00b8), sas_addr(0x5000cca26a235fa9), phy(26), device_name(0x5000cca26a235fab) [ 26.691687] scsi 2:0:211:0: enclosure logical id(0x5000ccab0405dc80), slot(35) [ 26.698991] scsi 2:0:211:0: enclosure level(0x0000), connector name( C0 ) [ 26.705869] scsi 2:0:211:0: serial_number( 2TGMG2KD) [ 26.711449] scsi 2:0:211:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.746623] mpt3sas_cm0: detecting: handle(0x00b9), sas_address(0x5000cca26a26b085), phy(27) [ 26.755061] mpt3sas_cm0: REPORT_LUNS: handle(0x00b9), retries(0) [ 26.761203] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00b9), lun(0) [ 26.767892] scsi 2:0:212:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.776281] scsi 2:0:212:0: SSP: handle(0x00b9), sas_addr(0x5000cca26a26b085), phy(27), device_name(0x5000cca26a26b087) [ 26.787054] scsi 2:0:212:0: enclosure logical id(0x5000ccab0405dc80), slot(36) [ 26.794358] scsi 2:0:212:0: enclosure level(0x0000), connector name( C0 ) [ 26.801253] scsi 2:0:212:0: serial_number( 2TGP8M1D) [ 26.806825] scsi 2:0:212:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.829631] mpt3sas_cm0: detecting: handle(0x00ba), sas_address(0x5000cca26a224ec9), phy(28) [ 26.838073] mpt3sas_cm0: REPORT_LUNS: handle(0x00ba), retries(0) [ 26.844216] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ba), lun(0) [ 26.850913] scsi 2:0:213:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.859301] scsi 2:0:213:0: SSP: handle(0x00ba), sas_addr(0x5000cca26a224ec9), phy(28), device_name(0x5000cca26a224ecb) [ 26.870073] scsi 2:0:213:0: enclosure logical id(0x5000ccab0405dc80), slot(37) [ 26.877378] scsi 2:0:213:0: enclosure level(0x0000), connector name( C0 ) [ 26.884269] scsi 2:0:213:0: serial_number( 2TGLVX6D) [ 26.889843] scsi 2:0:213:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.912696] mpt3sas_cm0: detecting: handle(0x00bb), sas_address(0x5000cca26a2c58d5), phy(29) [ 26.921134] mpt3sas_cm0: REPORT_LUNS: handle(0x00bb), retries(0) [ 26.927298] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bb), lun(0) [ 26.934124] scsi 2:0:214:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 26.942531] scsi 2:0:214:0: SSP: handle(0x00bb), sas_addr(0x5000cca26a2c58d5), phy(29), device_name(0x5000cca26a2c58d7) [ 26.953306] scsi 2:0:214:0: enclosure logical id(0x5000ccab0405dc80), slot(38) [ 26.960611] scsi 2:0:214:0: enclosure level(0x0000), connector name( C0 ) [ 26.967502] scsi 2:0:214:0: serial_number( 2TGTD23D) [ 26.973077] scsi 2:0:214:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 26.995620] mpt3sas_cm0: detecting: handle(0x00bc), sas_address(0x5000cca2525450fd), phy(30) [ 27.004062] mpt3sas_cm0: REPORT_LUNS: handle(0x00bc), retries(0) [ 27.010220] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bc), lun(0) [ 27.017058] scsi 2:0:215:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.025451] scsi 2:0:215:0: SSP: handle(0x00bc), sas_addr(0x5000cca2525450fd), phy(30), device_name(0x5000cca2525450ff) [ 27.036221] scsi 2:0:215:0: enclosure logical id(0x5000ccab0405dc80), slot(39) [ 27.043526] scsi 2:0:215:0: enclosure level(0x0000), connector name( C0 ) [ 27.050417] scsi 2:0:215:0: serial_number( 7SHHBHKG) [ 27.055992] scsi 2:0:215:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.078617] mpt3sas_cm0: detecting: handle(0x00bd), sas_address(0x5000cca266d33c69), phy(31) [ 27.087055] mpt3sas_cm0: REPORT_LUNS: handle(0x00bd), retries(0) [ 27.093229] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bd), lun(0) [ 27.099899] scsi 2:0:216:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.108291] scsi 2:0:216:0: SSP: handle(0x00bd), sas_addr(0x5000cca266d33c69), phy(31), device_name(0x5000cca266d33c6b) [ 27.119065] scsi 2:0:216:0: enclosure logical id(0x5000ccab0405dc80), slot(40) [ 27.126370] scsi 2:0:216:0: enclosure level(0x0000), connector name( C0 ) [ 27.133262] scsi 2:0:216:0: serial_number( 7JKS5AMK) [ 27.138836] scsi 2:0:216:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.161649] mpt3sas_cm0: detecting: handle(0x00be), sas_address(0x5000cca26a2435a9), phy(32) [ 27.170090] mpt3sas_cm0: REPORT_LUNS: handle(0x00be), retries(0) [ 27.176265] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00be), lun(0) [ 27.182926] scsi 2:0:217:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.191318] scsi 2:0:217:0: SSP: handle(0x00be), sas_addr(0x5000cca26a2435a9), phy(32), device_name(0x5000cca26a2435ab) [ 27.202090] scsi 2:0:217:0: enclosure logical id(0x5000ccab0405dc80), slot(41) [ 27.209398] scsi 2:0:217:0: enclosure level(0x0000), connector name( C0 ) [ 27.216290] scsi 2:0:217:0: serial_number( 2TGMXABD) [ 27.221863] scsi 2:0:217:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.244621] mpt3sas_cm0: detecting: handle(0x00bf), sas_address(0x5000cca268075749), phy(33) [ 27.253057] mpt3sas_cm0: REPORT_LUNS: handle(0x00bf), retries(0) [ 27.259228] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00bf), lun(0) [ 27.266291] scsi 2:0:218:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.274724] scsi 2:0:218:0: SSP: handle(0x00bf), sas_addr(0x5000cca268075749), phy(33), device_name(0x5000cca26807574b) [ 27.285498] scsi 2:0:218:0: enclosure logical id(0x5000ccab0405dc80), slot(42) [ 27.292805] scsi 2:0:218:0: enclosure level(0x0000), connector name( C0 ) [ 27.299696] scsi 2:0:218:0: serial_number( E4G414UK) [ 27.305271] scsi 2:0:218:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.328643] mpt3sas_cm0: detecting: handle(0x00c0), sas_address(0x5000cca26b9a57d1), phy(34) [ 27.337077] mpt3sas_cm0: REPORT_LUNS: handle(0x00c0), retries(0) [ 27.343361] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c0), lun(0) [ 27.367002] scsi 2:0:219:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.375475] scsi 2:0:219:0: SSP: handle(0x00c0), sas_addr(0x5000cca26b9a57d1), phy(34), device_name(0x5000cca26b9a57d3) [ 27.386248] scsi 2:0:219:0: enclosure logical id(0x5000ccab0405dc80), slot(43) [ 27.393555] scsi 2:0:219:0: enclosure level(0x0000), connector name( C0 ) [ 27.400433] scsi 2:0:219:0: serial_number( 1SJRWBTZ) [ 27.406011] scsi 2:0:219:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.431645] mpt3sas_cm0: detecting: handle(0x00c1), sas_address(0x5000cca26b9b2d01), phy(35) [ 27.440081] mpt3sas_cm0: REPORT_LUNS: handle(0x00c1), retries(0) [ 27.446208] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c1), lun(0) [ 27.452864] scsi 2:0:220:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.461249] scsi 2:0:220:0: SSP: handle(0x00c1), sas_addr(0x5000cca26b9b2d01), phy(35), device_name(0x5000cca26b9b2d03) [ 27.472022] scsi 2:0:220:0: enclosure logical id(0x5000ccab0405dc80), slot(44) [ 27.479328] scsi 2:0:220:0: enclosure level(0x0000), connector name( C0 ) [ 27.486224] scsi 2:0:220:0: serial_number( 1SJSAKXZ) [ 27.491804] scsi 2:0:220:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.511624] mpt3sas_cm0: detecting: handle(0x00c2), sas_address(0x5000cca26a2440e1), phy(36) [ 27.520058] mpt3sas_cm0: REPORT_LUNS: handle(0x00c2), retries(0) [ 27.526227] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c2), lun(0) [ 27.532902] scsi 2:0:221:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.541293] scsi 2:0:221:0: SSP: handle(0x00c2), sas_addr(0x5000cca26a2440e1), phy(36), device_name(0x5000cca26a2440e3) [ 27.552068] scsi 2:0:221:0: enclosure logical id(0x5000ccab0405dc80), slot(45) [ 27.559373] scsi 2:0:221:0: enclosure level(0x0000), connector name( C0 ) [ 27.566270] scsi 2:0:221:0: serial_number( 2TGMY2JD) [ 27.571852] scsi 2:0:221:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.595621] mpt3sas_cm0: detecting: handle(0x00c3), sas_address(0x5000cca25254dfed), phy(37) [ 27.604066] mpt3sas_cm0: REPORT_LUNS: handle(0x00c3), retries(0) [ 27.610230] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c3), lun(0) [ 27.634408] scsi 2:0:222:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.642812] scsi 2:0:222:0: SSP: handle(0x00c3), sas_addr(0x5000cca25254dfed), phy(37), device_name(0x5000cca25254dfef) [ 27.653580] scsi 2:0:222:0: enclosure logical id(0x5000ccab0405dc80), slot(46) [ 27.660887] scsi 2:0:222:0: enclosure level(0x0000), connector name( C0 ) [ 27.667761] scsi 2:0:222:0: serial_number( 7SHHP0NG) [ 27.673333] scsi 2:0:222:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.696635] mpt3sas_cm0: detecting: handle(0x00c4), sas_address(0x5000cca3c6194969), phy(38) [ 27.705074] mpt3sas_cm0: REPORT_LUNS: handle(0x00c4), retries(0) [ 27.711859] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c4), lun(0) [ 27.729415] scsi 2:0:223:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.737901] scsi 2:0:223:0: SSP: handle(0x00c4), sas_addr(0x5000cca3c6194969), phy(38), device_name(0x5000cca3c619496b) [ 27.748669] scsi 2:0:223:0: enclosure logical id(0x5000ccab0405dc80), slot(47) [ 27.755975] scsi 2:0:223:0: enclosure level(0x0000), connector name( C0 ) [ 27.762855] scsi 2:0:223:0: serial_number( 7G0EX3GK) [ 27.768436] scsi 2:0:223:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.788631] mpt3sas_cm0: detecting: handle(0x00c5), sas_address(0x5000cca2526b15f1), phy(39) [ 27.797072] mpt3sas_cm0: REPORT_LUNS: handle(0x00c5), retries(0) [ 27.803220] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c5), lun(0) [ 27.809871] scsi 2:0:224:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.818413] scsi 2:0:224:0: SSP: handle(0x00c5), sas_addr(0x5000cca2526b15f1), phy(39), device_name(0x5000cca2526b15f3) [ 27.829185] scsi 2:0:224:0: enclosure logical id(0x5000ccab0405dc80), slot(48) [ 27.836490] scsi 2:0:224:0: enclosure level(0x0000), connector name( C0 ) [ 27.843383] scsi 2:0:224:0: serial_number( 7SHWWPJU) [ 27.848956] scsi 2:0:224:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.871625] mpt3sas_cm0: detecting: handle(0x00c6), sas_address(0x5000cca26b9be875), phy(40) [ 27.880063] mpt3sas_cm0: REPORT_LUNS: handle(0x00c6), retries(0) [ 27.886215] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c6), lun(0) [ 27.892874] scsi 2:0:225:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.901264] scsi 2:0:225:0: SSP: handle(0x00c6), sas_addr(0x5000cca26b9be875), phy(40), device_name(0x5000cca26b9be877) [ 27.912038] scsi 2:0:225:0: enclosure logical id(0x5000ccab0405dc80), slot(49) [ 27.919343] scsi 2:0:225:0: enclosure level(0x0000), connector name( C0 ) [ 27.926234] scsi 2:0:225:0: serial_number( 1SJSS1XZ) [ 27.931808] scsi 2:0:225:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 27.951625] mpt3sas_cm0: detecting: handle(0x00c7), sas_address(0x5000cca26a259c0d), phy(41) [ 27.960063] mpt3sas_cm0: REPORT_LUNS: handle(0x00c7), retries(0) [ 27.966225] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c7), lun(0) [ 27.972990] scsi 2:0:226:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 27.989404] scsi 2:0:226:0: SSP: handle(0x00c7), sas_addr(0x5000cca26a259c0d), phy(41), device_name(0x5000cca26a259c0f) [ 28.000176] scsi 2:0:226:0: enclosure logical id(0x5000ccab0405dc80), slot(50) [ 28.007481] scsi 2:0:226:0: enclosure level(0x0000), connector name( C0 ) [ 28.014373] scsi 2:0:226:0: serial_number( 2TGNP68D) [ 28.019948] scsi 2:0:226:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.042632] mpt3sas_cm0: detecting: handle(0x00c8), sas_address(0x5000cca266d7197d), phy(42) [ 28.051070] mpt3sas_cm0: REPORT_LUNS: handle(0x00c8), retries(0) [ 28.057211] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c8), lun(0) [ 28.063922] scsi 2:0:227:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.072314] scsi 2:0:227:0: SSP: handle(0x00c8), sas_addr(0x5000cca266d7197d), phy(42), device_name(0x5000cca266d7197f) [ 28.083091] scsi 2:0:227:0: enclosure logical id(0x5000ccab0405dc80), slot(51) [ 28.090397] scsi 2:0:227:0: enclosure level(0x0000), connector name( C0 ) [ 28.097290] scsi 2:0:227:0: serial_number( 7JKU96LK) [ 28.102862] scsi 2:0:227:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.125679] mpt3sas_cm0: detecting: handle(0x00c9), sas_address(0x5000cca26a2223ad), phy(43) [ 28.134116] mpt3sas_cm0: REPORT_LUNS: handle(0x00c9), retries(0) [ 28.140249] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00c9), lun(0) [ 28.146934] scsi 2:0:228:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.155327] scsi 2:0:228:0: SSP: handle(0x00c9), sas_addr(0x5000cca26a2223ad), phy(43), device_name(0x5000cca26a2223af) [ 28.166100] scsi 2:0:228:0: enclosure logical id(0x5000ccab0405dc80), slot(52) [ 28.173404] scsi 2:0:228:0: enclosure level(0x0000), connector name( C0 ) [ 28.180299] scsi 2:0:228:0: serial_number( 2TGLT16D) [ 28.185874] scsi 2:0:228:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.209217] mpt3sas_cm0: detecting: handle(0x00ca), sas_address(0x5000cca266d5de41), phy(44) [ 28.217657] mpt3sas_cm0: REPORT_LUNS: handle(0x00ca), retries(0) [ 28.223797] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ca), lun(0) [ 28.230470] scsi 2:0:229:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.238858] scsi 2:0:229:0: SSP: handle(0x00ca), sas_addr(0x5000cca266d5de41), phy(44), device_name(0x5000cca266d5de43) [ 28.249629] scsi 2:0:229:0: enclosure logical id(0x5000ccab0405dc80), slot(53) [ 28.256934] scsi 2:0:229:0: enclosure level(0x0000), connector name( C0 ) [ 28.263825] scsi 2:0:229:0: serial_number( 7JKTM6TK) [ 28.269399] scsi 2:0:229:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.289625] mpt3sas_cm0: detecting: handle(0x00cb), sas_address(0x5000cca26b9d3155), phy(45) [ 28.298061] mpt3sas_cm0: REPORT_LUNS: handle(0x00cb), retries(0) [ 28.304199] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00cb), lun(0) [ 28.310921] scsi 2:0:230:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.319308] scsi 2:0:230:0: SSP: handle(0x00cb), sas_addr(0x5000cca26b9d3155), phy(45), device_name(0x5000cca26b9d3157) [ 28.330081] scsi 2:0:230:0: enclosure logical id(0x5000ccab0405dc80), slot(54) [ 28.337388] scsi 2:0:230:0: enclosure level(0x0000), connector name( C0 ) [ 28.344279] scsi 2:0:230:0: serial_number( 1SJTEYWZ) [ 28.349852] scsi 2:0:230:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.372643] mpt3sas_cm0: detecting: handle(0x00cc), sas_address(0x5000cca26a24fce5), phy(46) [ 28.381082] mpt3sas_cm0: REPORT_LUNS: handle(0x00cc), retries(0) [ 28.387246] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00cc), lun(0) [ 28.394177] scsi 2:0:231:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.406835] scsi 2:0:231:0: SSP: handle(0x00cc), sas_addr(0x5000cca26a24fce5), phy(46), device_name(0x5000cca26a24fce7) [ 28.417607] scsi 2:0:231:0: enclosure logical id(0x5000ccab0405dc80), slot(55) [ 28.424911] scsi 2:0:231:0: enclosure level(0x0000), connector name( C0 ) [ 28.431804] scsi 2:0:231:0: serial_number( 2TGNALPD) [ 28.437376] scsi 2:0:231:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.466658] mpt3sas_cm0: detecting: handle(0x00cd), sas_address(0x5000cca26a26f9a9), phy(47) [ 28.475096] mpt3sas_cm0: REPORT_LUNS: handle(0x00cd), retries(0) [ 28.481234] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00cd), lun(0) [ 28.487917] scsi 2:0:232:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.496313] scsi 2:0:232:0: SSP: handle(0x00cd), sas_addr(0x5000cca26a26f9a9), phy(47), device_name(0x5000cca26a26f9ab) [ 28.507088] scsi 2:0:232:0: enclosure logical id(0x5000ccab0405dc80), slot(56) [ 28.514395] scsi 2:0:232:0: enclosure level(0x0000), connector name( C0 ) [ 28.521288] scsi 2:0:232:0: serial_number( 2TGPEH1D) [ 28.526861] scsi 2:0:232:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.549636] mpt3sas_cm0: detecting: handle(0x00ce), sas_address(0x5000cca26a261b15), phy(48) [ 28.558084] mpt3sas_cm0: REPORT_LUNS: handle(0x00ce), retries(0) [ 28.564224] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00ce), lun(0) [ 28.570899] scsi 2:0:233:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.579292] scsi 2:0:233:0: SSP: handle(0x00ce), sas_addr(0x5000cca26a261b15), phy(48), device_name(0x5000cca26a261b17) [ 28.590063] scsi 2:0:233:0: enclosure logical id(0x5000ccab0405dc80), slot(57) [ 28.597370] scsi 2:0:233:0: enclosure level(0x0000), connector name( C0 ) [ 28.604266] scsi 2:0:233:0: serial_number( 2TGNYNJD) [ 28.609846] scsi 2:0:233:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.632631] mpt3sas_cm0: detecting: handle(0x00cf), sas_address(0x5000cca26a23dffd), phy(49) [ 28.641073] mpt3sas_cm0: REPORT_LUNS: handle(0x00cf), retries(0) [ 28.647212] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00cf), lun(0) [ 28.657642] scsi 2:0:234:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.666019] scsi 2:0:234:0: SSP: handle(0x00cf), sas_addr(0x5000cca26a23dffd), phy(49), device_name(0x5000cca26a23dfff) [ 28.676791] scsi 2:0:234:0: enclosure logical id(0x5000ccab0405dc80), slot(58) [ 28.684098] scsi 2:0:234:0: enclosure level(0x0000), connector name( C0 ) [ 28.690989] scsi 2:0:234:0: serial_number( 2TGMRMHD) [ 28.696561] scsi 2:0:234:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.716672] mpt3sas_cm0: detecting: handle(0x00d0), sas_address(0x5000cca26a258edd), phy(50) [ 28.725113] mpt3sas_cm0: REPORT_LUNS: handle(0x00d0), retries(0) [ 28.731282] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d0), lun(0) [ 28.738039] scsi 2:0:235:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.746428] scsi 2:0:235:0: SSP: handle(0x00d0), sas_addr(0x5000cca26a258edd), phy(50), device_name(0x5000cca26a258edf) [ 28.757199] scsi 2:0:235:0: enclosure logical id(0x5000ccab0405dc80), slot(59) [ 28.764505] scsi 2:0:235:0: enclosure level(0x0000), connector name( C0 ) [ 28.771399] scsi 2:0:235:0: serial_number( 2TGNNA1D) [ 28.776972] scsi 2:0:235:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.802120] mpt3sas_cm0: expander_add: handle(0x009c), parent(0x0099), sas_addr(0x5000ccab0405dcff), phys(68) [ 28.824419] mpt3sas_cm0: detecting: handle(0x00d1), sas_address(0x5000cca25254e76d), phy(42) [ 28.832856] mpt3sas_cm0: REPORT_LUNS: handle(0x00d1), retries(0) [ 28.838984] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d1), lun(0) [ 28.849883] scsi 2:0:236:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.858263] scsi 2:0:236:0: SSP: handle(0x00d1), sas_addr(0x5000cca25254e76d), phy(42), device_name(0x5000cca25254e76f) [ 28.869034] scsi 2:0:236:0: enclosure logical id(0x5000ccab0405dc80), slot(1) [ 28.876254] scsi 2:0:236:0: enclosure level(0x0000), connector name( C0 ) [ 28.883133] scsi 2:0:236:0: serial_number( 7SHHPJ4G) [ 28.888711] scsi 2:0:236:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.918637] mpt3sas_cm0: detecting: handle(0x00d2), sas_address(0x5000cca25255997d), phy(43) [ 28.927070] mpt3sas_cm0: REPORT_LUNS: handle(0x00d2), retries(0) [ 28.933203] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d2), lun(0) [ 28.939864] scsi 2:0:237:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 28.948266] scsi 2:0:237:0: SSP: handle(0x00d2), sas_addr(0x5000cca25255997d), phy(43), device_name(0x5000cca25255997f) [ 28.959039] scsi 2:0:237:0: enclosure logical id(0x5000ccab0405dc80), slot(3) [ 28.966256] scsi 2:0:237:0: enclosure level(0x0000), connector name( C0 ) [ 28.973150] scsi 2:0:237:0: serial_number( 7SHJ2BSG) [ 28.978723] scsi 2:0:237:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 28.999259] mpt3sas_cm0: detecting: handle(0x00d3), sas_address(0x5000cca25255a069), phy(44) [ 29.007701] mpt3sas_cm0: REPORT_LUNS: handle(0x00d3), retries(0) [ 29.013866] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d3), lun(0) [ 29.020657] scsi 2:0:238:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 29.029054] scsi 2:0:238:0: SSP: handle(0x00d3), sas_addr(0x5000cca25255a069), phy(44), device_name(0x5000cca25255a06b) [ 29.039827] scsi 2:0:238:0: enclosure logical id(0x5000ccab0405dc80), slot(4) [ 29.047046] scsi 2:0:238:0: enclosure level(0x0000), connector name( C0 ) [ 29.053939] scsi 2:0:238:0: serial_number( 7SHJ2V1G) [ 29.059513] scsi 2:0:238:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.079635] mpt3sas_cm0: detecting: handle(0x00d4), sas_address(0x5000cca252561fbd), phy(45) [ 29.088073] mpt3sas_cm0: REPORT_LUNS: handle(0x00d4), retries(0) [ 29.094214] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d4), lun(0) [ 29.101045] scsi 2:0:239:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 29.109442] scsi 2:0:239:0: SSP: handle(0x00d4), sas_addr(0x5000cca252561fbd), phy(45), device_name(0x5000cca252561fbf) [ 29.120212] scsi 2:0:239:0: enclosure logical id(0x5000ccab0405dc80), slot(5) [ 29.127431] scsi 2:0:239:0: enclosure level(0x0000), connector name( C0 ) [ 29.134323] scsi 2:0:239:0: serial_number( 7SHJB9XG) [ 29.139895] scsi 2:0:239:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.162637] mpt3sas_cm0: detecting: handle(0x00d5), sas_address(0x5000cca25254f76d), phy(46) [ 29.171075] mpt3sas_cm0: REPORT_LUNS: handle(0x00d5), retries(0) [ 29.177217] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d5), lun(0) [ 29.184039] scsi 2:0:240:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 29.192439] scsi 2:0:240:0: SSP: handle(0x00d5), sas_addr(0x5000cca25254f76d), phy(46), device_name(0x5000cca25254f76f) [ 29.203211] scsi 2:0:240:0: enclosure logical id(0x5000ccab0405dc80), slot(6) [ 29.210431] scsi 2:0:240:0: enclosure level(0x0000), connector name( C0 ) [ 29.217324] scsi 2:0:240:0: serial_number( 7SHHRL5G) [ 29.222897] scsi 2:0:240:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.245638] mpt3sas_cm0: detecting: handle(0x00d6), sas_address(0x5000cca252544e65), phy(47) [ 29.254074] mpt3sas_cm0: REPORT_LUNS: handle(0x00d6), retries(0) [ 29.260215] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d6), lun(0) [ 29.267022] scsi 2:0:241:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 29.275420] scsi 2:0:241:0: SSP: handle(0x00d6), sas_addr(0x5000cca252544e65), phy(47), device_name(0x5000cca252544e67) [ 29.286194] scsi 2:0:241:0: enclosure logical id(0x5000ccab0405dc80), slot(7) [ 29.293414] scsi 2:0:241:0: enclosure level(0x0000), connector name( C0 ) [ 29.300308] scsi 2:0:241:0: serial_number( 7SHHBA6G) [ 29.305881] scsi 2:0:241:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.328637] mpt3sas_cm0: detecting: handle(0x00d7), sas_address(0x5000cca252562c29), phy(48) [ 29.337073] mpt3sas_cm0: REPORT_LUNS: handle(0x00d7), retries(0) [ 29.343202] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d7), lun(0) [ 29.349880] scsi 2:0:242:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 29.358265] scsi 2:0:242:0: SSP: handle(0x00d7), sas_addr(0x5000cca252562c29), phy(48), device_name(0x5000cca252562c2b) [ 29.369040] scsi 2:0:242:0: enclosure logical id(0x5000ccab0405dc80), slot(8) [ 29.376257] scsi 2:0:242:0: enclosure level(0x0000), connector name( C0 ) [ 29.383152] scsi 2:0:242:0: serial_number( 7SHJD4KG) [ 29.388725] scsi 2:0:242:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.408638] mpt3sas_cm0: detecting: handle(0x00d8), sas_address(0x5000cca25255d15d), phy(49) [ 29.417075] mpt3sas_cm0: REPORT_LUNS: handle(0x00d8), retries(0) [ 29.423242] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d8), lun(0) [ 29.429899] scsi 2:0:243:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 29.438286] scsi 2:0:243:0: SSP: handle(0x00d8), sas_addr(0x5000cca25255d15d), phy(49), device_name(0x5000cca25255d15f) [ 29.449060] scsi 2:0:243:0: enclosure logical id(0x5000ccab0405dc80), slot(9) [ 29.456277] scsi 2:0:243:0: enclosure level(0x0000), connector name( C0 ) [ 29.463169] scsi 2:0:243:0: serial_number( 7SHJ633G) [ 29.468744] scsi 2:0:243:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.488639] mpt3sas_cm0: detecting: handle(0x00d9), sas_address(0x5000cca25253e25d), phy(50) [ 29.497077] mpt3sas_cm0: REPORT_LUNS: handle(0x00d9), retries(0) [ 29.503219] mpt3sas_cm0: TEST_UNIT_READY: handle(0x00d9), lun(0) [ 29.513987] scsi 2:0:244:0: Direct-Access HGST HUH721008AL5200 A38F PQ: 0 ANSI: 6 [ 29.522456] scsi 2:0:244:0: SSP: handle(0x00d9), sas_addr(0x5000cca25253e25d), phy(50), device_name(0x5000cca25253e25f) [ 29.533230] scsi 2:0:244:0: enclosure logical id(0x5000ccab0405dc80), slot(10) [ 29.540536] scsi 2:0:244:0: enclosure level(0x0000), connector name( C0 ) [ 29.547413] scsi 2:0:244:0: serial_number( 7SHH445G) [ 29.552996] scsi 2:0:244:0: qdepth(254), tagged(1), simple(0), ordered(0), scsi_level(7), cmd_que(1) [ 29.583449] mpt3sas_cm0: port enable: SUCCESS [ 29.588577] sd 2:0:2:0: [sdb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.596428] sd 2:0:2:0: [sdb] 4096-byte physical blocks [ 29.596440] sd 2:0:3:0: [sdc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.596442] sd 2:0:3:0: [sdc] 4096-byte physical blocks [ 29.609524] sd 2:0:4:0: [sdd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.609526] sd 2:0:5:0: [sde] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.609528] sd 2:0:4:0: [sdd] 4096-byte physical blocks [ 29.609528] sd 2:0:5:0: [sde] 4096-byte physical blocks [ 29.622793] sd 2:0:9:0: [sdi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.622796] sd 2:0:10:0: [sdj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.622798] sd 2:0:9:0: [sdi] 4096-byte physical blocks [ 29.622800] sd 2:0:10:0: [sdj] 4096-byte physical blocks [ 29.622804] sd 2:0:6:0: [sdf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.622806] sd 2:0:6:0: [sdf] 4096-byte physical blocks [ 29.622815] sd 2:0:7:0: [sdg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.622817] sd 2:0:7:0: [sdg] 4096-byte physical blocks [ 29.622826] sd 2:0:8:0: [sdh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.622827] sd 2:0:8:0: [sdh] 4096-byte physical blocks [ 29.630708] sd 2:0:13:0: [sdm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630710] sd 2:0:13:0: [sdm] 4096-byte physical blocks [ 29.630721] sd 2:0:39:0: [sdam] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630723] sd 2:0:39:0: [sdam] 4096-byte physical blocks [ 29.630725] sd 2:0:40:0: [sdan] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630727] sd 2:0:40:0: [sdan] 4096-byte physical blocks [ 29.630730] sd 2:0:29:0: [sdac] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630731] sd 2:0:28:0: [sdab] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630732] sd 2:0:29:0: [sdac] 4096-byte physical blocks [ 29.630733] sd 2:0:28:0: [sdab] 4096-byte physical blocks [ 29.630736] sd 2:0:14:0: [sdn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630738] sd 2:0:14:0: [sdn] 4096-byte physical blocks [ 29.630746] sd 2:0:30:0: [sdad] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630748] sd 2:0:30:0: [sdad] 4096-byte physical blocks [ 29.630752] sd 2:0:25:0: [sdy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630754] sd 2:0:18:0: [sdr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630756] sd 2:0:25:0: [sdy] 4096-byte physical blocks [ 29.630758] sd 2:0:18:0: [sdr] 4096-byte physical blocks [ 29.630759] sd 2:0:35:0: [sdai] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630760] sd 2:0:35:0: [sdai] 4096-byte physical blocks [ 29.630764] sd 2:0:12:0: [sdl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630766] sd 2:0:12:0: [sdl] 4096-byte physical blocks [ 29.630777] sd 2:0:46:0: [sdat] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630779] sd 2:0:46:0: [sdat] 4096-byte physical blocks [ 29.630782] sd 2:0:51:0: [sday] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630784] sd 2:0:51:0: [sday] 4096-byte physical blocks [ 29.630789] sd 2:0:47:0: [sdau] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630791] sd 2:0:47:0: [sdau] 4096-byte physical blocks [ 29.630805] sd 2:0:16:0: [sdp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630807] sd 2:0:52:0: [sdaz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630809] sd 2:0:16:0: [sdp] 4096-byte physical blocks [ 29.630810] sd 2:0:22:0: [sdv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630812] sd 2:0:52:0: [sdaz] 4096-byte physical blocks [ 29.630814] sd 2:0:22:0: [sdv] 4096-byte physical blocks [ 29.630818] sd 2:0:15:0: [sdo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630820] sd 2:0:15:0: [sdo] 4096-byte physical blocks [ 29.630821] sd 2:0:41:0: [sdao] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630823] sd 2:0:41:0: [sdao] 4096-byte physical blocks [ 29.630829] sd 2:0:34:0: [sdah] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630830] sd 2:0:34:0: [sdah] 4096-byte physical blocks [ 29.630836] sd 2:0:36:0: [sdaj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630837] sd 2:0:21:0: [sdu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630837] sd 2:0:36:0: [sdaj] 4096-byte physical blocks [ 29.630838] sd 2:0:21:0: [sdu] 4096-byte physical blocks [ 29.630843] sd 2:0:43:0: [sdaq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630845] sd 2:0:43:0: [sdaq] 4096-byte physical blocks [ 29.630849] sd 2:0:45:0: [sdas] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630850] sd 2:0:45:0: [sdas] 4096-byte physical blocks [ 29.630853] sd 2:0:26:0: [sdz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630854] sd 2:0:48:0: [sdav] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630854] sd 2:0:26:0: [sdz] 4096-byte physical blocks [ 29.630855] sd 2:0:48:0: [sdav] 4096-byte physical blocks [ 29.630857] sd 2:0:17:0: [sdq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630859] sd 2:0:17:0: [sdq] 4096-byte physical blocks [ 29.630867] sd 2:0:27:0: [sdaa] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630868] sd 2:0:31:0: [sdae] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630869] sd 2:0:27:0: [sdaa] 4096-byte physical blocks [ 29.630871] sd 2:0:23:0: [sdw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630872] sd 2:0:31:0: [sdae] 4096-byte physical blocks [ 29.630874] sd 2:0:23:0: [sdw] 4096-byte physical blocks [ 29.630875] sd 2:0:32:0: [sdaf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630877] sd 2:0:32:0: [sdaf] 4096-byte physical blocks [ 29.630881] sd 2:0:33:0: [sdag] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630882] sd 2:0:33:0: [sdag] 4096-byte physical blocks [ 29.630885] sd 2:0:19:0: [sds] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630886] sd 2:0:20:0: [sdt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630886] sd 2:0:19:0: [sds] 4096-byte physical blocks [ 29.630887] sd 2:0:20:0: [sdt] 4096-byte physical blocks [ 29.630891] sd 2:0:44:0: [sdar] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630892] sd 2:0:44:0: [sdar] 4096-byte physical blocks [ 29.630912] sd 2:0:38:0: [sdal] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630914] sd 2:0:38:0: [sdal] 4096-byte physical blocks [ 29.630921] sd 2:0:50:0: [sdax] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.630922] sd 2:0:50:0: [sdax] 4096-byte physical blocks [ 29.632473] sd 2:0:49:0: [sdaw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.632475] sd 2:0:49:0: [sdaw] 4096-byte physical blocks [ 29.632484] sd 2:0:42:0: [sdap] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.632486] sd 2:0:42:0: [sdap] 4096-byte physical blocks [ 29.636208] sd 2:0:53:0: [sdba] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636210] sd 2:0:53:0: [sdba] 4096-byte physical blocks [ 29.636234] sd 2:0:55:0: [sdbc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636237] sd 2:0:55:0: [sdbc] 4096-byte physical blocks [ 29.636243] sd 2:0:60:0: [sdbh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636245] sd 2:0:60:0: [sdbh] 4096-byte physical blocks [ 29.636268] sd 2:0:68:0: [sdbo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636269] sd 2:0:68:0: [sdbo] 4096-byte physical blocks [ 29.636298] sd 2:0:54:0: [sdbb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636299] sd 2:0:54:0: [sdbb] 4096-byte physical blocks [ 29.636318] sd 2:0:58:0: [sdbf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636320] sd 2:0:64:0: [sdbk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636322] sd 2:0:58:0: [sdbf] 4096-byte physical blocks [ 29.636324] sd 2:0:64:0: [sdbk] 4096-byte physical blocks [ 29.636328] sd 2:0:65:0: [sdbl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636329] sd 2:0:65:0: [sdbl] 4096-byte physical blocks [ 29.636331] sd 2:0:69:0: [sdbp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636332] sd 2:0:69:0: [sdbp] 4096-byte physical blocks [ 29.636347] sd 2:0:24:0: [sdx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636348] sd 2:0:56:0: [sdbd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636350] sd 2:0:24:0: [sdx] 4096-byte physical blocks [ 29.636352] sd 2:0:56:0: [sdbd] 4096-byte physical blocks [ 29.636366] sd 2:0:57:0: [sdbe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636368] sd 2:0:57:0: [sdbe] 4096-byte physical blocks [ 29.636375] sd 2:0:66:0: [sdbm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636377] sd 2:0:66:0: [sdbm] 4096-byte physical blocks [ 29.636439] sd 2:0:59:0: [sdbg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636441] sd 2:0:59:0: [sdbg] 4096-byte physical blocks [ 29.636455] sd 2:0:67:0: [sdbn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.636456] sd 2:0:67:0: [sdbn] 4096-byte physical blocks [ 29.641466] sd 2:0:63:0: [sdbj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.641468] sd 2:0:63:0: [sdbj] 4096-byte physical blocks [ 29.641829] sd 2:0:61:0: [sdbi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.641831] sd 2:0:61:0: [sdbi] 4096-byte physical blocks [ 29.641908] sd 2:0:3:0: [sdc] Write Protect is off [ 29.641910] sd 2:0:3:0: [sdc] Mode Sense: f7 00 10 08 [ 29.641916] sd 2:0:70:0: [sdbq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.641917] sd 2:0:70:0: [sdbq] 4096-byte physical blocks [ 29.641924] sd 2:0:11:0: [sdk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.641926] sd 2:0:11:0: [sdk] 4096-byte physical blocks [ 29.641936] sd 2:0:71:0: [sdbr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.641937] sd 2:0:71:0: [sdbr] 4096-byte physical blocks [ 29.649946] sd 2:0:4:0: [sdd] Write Protect is off [ 29.649948] sd 2:0:4:0: [sdd] Mode Sense: f7 00 10 08 [ 29.650024] sd 2:0:72:0: [sdbs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.650026] sd 2:0:72:0: [sdbs] 4096-byte physical blocks [ 29.650056] sd 2:0:5:0: [sde] Write Protect is off [ 29.650058] sd 2:0:5:0: [sde] Mode Sense: f7 00 10 08 [ 29.658190] sd 2:0:3:0: [sdc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.658199] sd 2:0:8:0: [sdh] Write Protect is off [ 29.658201] sd 2:0:73:0: [sdbt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.658202] sd 2:0:8:0: [sdh] Mode Sense: f7 00 10 08 [ 29.658202] sd 2:0:73:0: [sdbt] 4096-byte physical blocks [ 29.658227] sd 2:0:10:0: [sdj] Write Protect is off [ 29.658228] sd 2:0:6:0: [sdf] Write Protect is off [ 29.658230] sd 2:0:10:0: [sdj] Mode Sense: f7 00 10 08 [ 29.658232] sd 2:0:6:0: [sdf] Mode Sense: f7 00 10 08 [ 29.658257] sd 2:0:9:0: [sdi] Write Protect is off [ 29.658259] sd 2:0:9:0: [sdi] Mode Sense: f7 00 10 08 [ 29.658283] sd 2:0:74:0: [sdbu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.658285] sd 2:0:74:0: [sdbu] 4096-byte physical blocks [ 29.658305] sd 2:0:75:0: [sdbv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.658307] sd 2:0:75:0: [sdbv] 4096-byte physical blocks [ 29.662466] sd 2:0:7:0: [sdg] Write Protect is off [ 29.662468] sd 2:0:7:0: [sdg] Mode Sense: f7 00 10 08 [ 29.663623] sd 2:0:15:0: [sdo] Write Protect is off [ 29.663626] sd 2:0:15:0: [sdo] Mode Sense: f7 00 10 08 [ 29.663631] sd 2:0:25:0: [sdy] Write Protect is off [ 29.663633] sd 2:0:25:0: [sdy] Mode Sense: f7 00 10 08 [ 29.663634] sd 2:0:19:0: [sds] Write Protect is off [ 29.663635] sd 2:0:44:0: [sdar] Write Protect is off [ 29.663636] sd 2:0:19:0: [sds] Mode Sense: f7 00 10 08 [ 29.663636] sd 2:0:44:0: [sdar] Mode Sense: f7 00 10 08 [ 29.663643] sd 2:0:20:0: [sdt] Write Protect is off [ 29.663644] sd 2:0:20:0: [sdt] Mode Sense: f7 00 10 08 [ 29.663650] sd 2:0:49:0: [sdaw] Write Protect is off [ 29.663652] sd 2:0:49:0: [sdaw] Mode Sense: f7 00 10 08 [ 29.663653] sd 2:0:12:0: [sdl] Write Protect is off [ 29.663655] sd 2:0:12:0: [sdl] Mode Sense: f7 00 10 08 [ 29.663666] sd 2:0:38:0: [sdal] Write Protect is off [ 29.663667] sd 2:0:38:0: [sdal] Mode Sense: f7 00 10 08 [ 29.663679] sd 2:0:32:0: [sdaf] Write Protect is off [ 29.663679] sd 2:0:40:0: [sdan] Write Protect is off [ 29.663680] sd 2:0:32:0: [sdaf] Mode Sense: f7 00 10 08 [ 29.663681] sd 2:0:40:0: [sdan] Mode Sense: f7 00 10 08 [ 29.663685] sd 2:0:29:0: [sdac] Write Protect is off [ 29.663687] sd 2:0:29:0: [sdac] Mode Sense: f7 00 10 08 [ 29.663700] sd 2:0:14:0: [sdn] Write Protect is off [ 29.663701] sd 2:0:4:0: [sdd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.663704] sd 2:0:41:0: [sdao] Write Protect is off [ 29.663705] sd 2:0:14:0: [sdn] Mode Sense: f7 00 10 08 [ 29.663707] sd 2:0:46:0: [sdat] Write Protect is off [ 29.663709] sd 2:0:43:0: [sdaq] Write Protect is off [ 29.663711] sd 2:0:13:0: [sdm] Write Protect is off [ 29.663713] sd 2:0:41:0: [sdao] Mode Sense: f7 00 10 08 [ 29.663715] sd 2:0:46:0: [sdat] Mode Sense: f7 00 10 08 [ 29.663718] sd 2:0:43:0: [sdaq] Mode Sense: f7 00 10 08 [ 29.663719] sd 2:0:48:0: [sdav] Write Protect is off [ 29.663721] sd 2:0:13:0: [sdm] Mode Sense: f7 00 10 08 [ 29.663722] sd 2:0:28:0: [sdab] Write Protect is off [ 29.663723] sd 2:0:48:0: [sdav] Mode Sense: f7 00 10 08 [ 29.663725] sd 2:0:51:0: [sday] Write Protect is off [ 29.663727] sd 2:0:28:0: [sdab] Mode Sense: f7 00 10 08 [ 29.663729] sd 2:0:18:0: [sdr] Write Protect is off [ 29.663730] sd 2:0:51:0: [sday] Mode Sense: f7 00 10 08 [ 29.663732] sd 2:0:18:0: [sdr] Mode Sense: f7 00 10 08 [ 29.663735] sd 2:0:47:0: [sdau] Write Protect is off [ 29.663736] sd 2:0:31:0: [sdae] Write Protect is off [ 29.663738] sd 2:0:52:0: [sdaz] Write Protect is off [ 29.663740] sd 2:0:47:0: [sdau] Mode Sense: f7 00 10 08 [ 29.663741] sd 2:0:31:0: [sdae] Mode Sense: f7 00 10 08 [ 29.663743] sd 2:0:50:0: [sdax] Write Protect is off [ 29.663744] sd 2:0:23:0: [sdw] Write Protect is off [ 29.663745] sd 2:0:52:0: [sdaz] Mode Sense: f7 00 10 08 [ 29.663747] sd 2:0:30:0: [sdad] Write Protect is off [ 29.663748] sd 2:0:50:0: [sdax] Mode Sense: f7 00 10 08 [ 29.663750] sd 2:0:23:0: [sdw] Mode Sense: f7 00 10 08 [ 29.663752] sd 2:0:35:0: [sdai] Write Protect is off [ 29.663753] sd 2:0:22:0: [sdv] Write Protect is off [ 29.663755] sd 2:0:76:0: [sdbw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.663757] sd 2:0:30:0: [sdad] Mode Sense: f7 00 10 08 [ 29.663758] sd 2:0:27:0: [sdaa] Write Protect is off [ 29.663759] sd 2:0:16:0: [sdp] Write Protect is off [ 29.663760] sd 2:0:35:0: [sdai] Mode Sense: f7 00 10 08 [ 29.663761] sd 2:0:22:0: [sdv] Mode Sense: f7 00 10 08 [ 29.663763] sd 2:0:76:0: [sdbw] 4096-byte physical blocks [ 29.663765] sd 2:0:27:0: [sdaa] Mode Sense: f7 00 10 08 [ 29.663766] sd 2:0:16:0: [sdp] Mode Sense: f7 00 10 08 [ 29.663768] sd 2:0:39:0: [sdam] Write Protect is off [ 29.663769] sd 2:0:39:0: [sdam] Mode Sense: f7 00 10 08 [ 29.663771] sd 2:0:45:0: [sdas] Write Protect is off [ 29.663773] sd 2:0:42:0: [sdap] Write Protect is off [ 29.663775] sd 2:0:36:0: [sdaj] Write Protect is off [ 29.663777] sd 2:0:21:0: [sdu] Write Protect is off [ 29.663778] sd 2:0:45:0: [sdas] Mode Sense: f7 00 10 08 [ 29.663780] sd 2:0:42:0: [sdap] Mode Sense: f7 00 10 08 [ 29.663781] sd 2:0:33:0: [sdag] Write Protect is off [ 29.663782] sd 2:0:36:0: [sdaj] Mode Sense: f7 00 10 08 [ 29.663785] sd 2:0:21:0: [sdu] Mode Sense: f7 00 10 08 [ 29.663787] sd 2:0:33:0: [sdag] Mode Sense: f7 00 10 08 [ 29.663788] sd 2:0:78:0: [sdby] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.663790] sd 2:0:78:0: [sdby] 4096-byte physical blocks [ 29.663796] sd 2:0:34:0: [sdah] Write Protect is off [ 29.663798] sd 2:0:77:0: [sdbx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.663800] sd 2:0:34:0: [sdah] Mode Sense: f7 00 10 08 [ 29.663802] sd 2:0:77:0: [sdbx] 4096-byte physical blocks [ 29.663813] sd 2:0:17:0: [sdq] Write Protect is off [ 29.663814] sd 2:0:17:0: [sdq] Mode Sense: f7 00 10 08 [ 29.665470] sd 2:0:26:0: [sdz] Write Protect is off [ 29.665472] sd 2:0:26:0: [sdz] Mode Sense: f7 00 10 08 [ 29.665480] sd 2:0:37:0: [sdak] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.665481] sd 2:0:37:0: [sdak] 4096-byte physical blocks [ 29.668461] sd 2:0:5:0: [sde] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.669176] sd 2:0:69:0: [sdbp] Write Protect is off [ 29.669178] sd 2:0:69:0: [sdbp] Mode Sense: f7 00 10 08 [ 29.669189] sd 2:0:55:0: [sdbc] Write Protect is off [ 29.669190] sd 2:0:55:0: [sdbc] Mode Sense: f7 00 10 08 [ 29.669199] sd 2:0:66:0: [sdbm] Write Protect is off [ 29.669201] sd 2:0:66:0: [sdbm] Mode Sense: f7 00 10 08 [ 29.669215] sd 2:0:8:0: [sdh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.669219] sd 2:0:68:0: [sdbo] Write Protect is off [ 29.669222] sd 2:0:68:0: [sdbo] Mode Sense: f7 00 10 08 [ 29.669222] sd 2:0:56:0: [sdbd] Write Protect is off [ 29.669224] sd 2:0:56:0: [sdbd] Mode Sense: f7 00 10 08 [ 29.669231] sd 2:0:6:0: [sdf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.669239] sd 2:0:57:0: [sdbe] Write Protect is off [ 29.669240] sd 2:0:57:0: [sdbe] Mode Sense: f7 00 10 08 [ 29.669258] sd 2:0:54:0: [sdbb] Write Protect is off [ 29.669259] sd 2:0:54:0: [sdbb] Mode Sense: f7 00 10 08 [ 29.669272] sd 2:0:64:0: [sdbk] Write Protect is off [ 29.669274] sd 2:0:58:0: [sdbf] Write Protect is off [ 29.669276] sd 2:0:64:0: [sdbk] Mode Sense: f7 00 10 08 [ 29.669278] sd 2:0:58:0: [sdbf] Mode Sense: f7 00 10 08 [ 29.669282] sd 2:0:60:0: [sdbh] Write Protect is off [ 29.669284] sd 2:0:60:0: [sdbh] Mode Sense: f7 00 10 08 [ 29.669311] sd 2:0:7:0: [sdg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.669313] sd 2:0:67:0: [sdbn] Write Protect is off [ 29.669314] sd 2:0:67:0: [sdbn] Mode Sense: f7 00 10 08 [ 29.669315] sd 2:0:79:0: [sdbz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.669317] sd 2:0:79:0: [sdbz] 4096-byte physical blocks [ 29.669319] sd 2:0:65:0: [sdbl] Write Protect is off [ 29.669320] sd 2:0:65:0: [sdbl] Mode Sense: f7 00 10 08 [ 29.669324] sd 2:0:9:0: [sdi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.669327] sd 2:0:24:0: [sdx] Write Protect is off [ 29.669329] sd 2:0:24:0: [sdx] Mode Sense: f7 00 10 08 [ 29.669335] sd 2:0:10:0: [sdj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.669337] sd 2:0:59:0: [sdbg] Write Protect is off [ 29.669339] sd 2:0:59:0: [sdbg] Mode Sense: f7 00 10 08 [ 29.669359] sd 2:0:53:0: [sdba] Write Protect is off [ 29.669361] sd 2:0:80:0: [sdca] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.669362] sd 2:0:53:0: [sdba] Mode Sense: f7 00 10 08 [ 29.669363] sd 2:0:80:0: [sdca] 4096-byte physical blocks [ 29.669366] sd 2:0:63:0: [sdbj] Write Protect is off [ 29.669367] sd 2:0:63:0: [sdbj] Mode Sense: f7 00 10 08 [ 29.677271] sd 2:0:35:0: [sdai] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677275] sd 2:0:34:0: [sdah] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677281] sd 2:0:45:0: [sdas] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677286] sd 2:0:61:0: [sdbi] Write Protect is off [ 29.677288] sd 2:0:61:0: [sdbi] Mode Sense: f7 00 10 08 [ 29.677292] sd 2:0:46:0: [sdat] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677293] sd 2:0:31:0: [sdae] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677303] sd 2:0:30:0: [sdad] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677305] sd 2:0:15:0: [sdo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677308] sd 2:0:33:0: [sdag] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677310] sd 2:0:70:0: [sdbq] Write Protect is off [ 29.677312] sd 2:0:27:0: [sdaa] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677314] sd 2:0:70:0: [sdbq] Mode Sense: f7 00 10 08 [ 29.677315] sd 2:0:51:0: [sday] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677319] sd 2:0:49:0: [sdaw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677321] sd 2:0:40:0: [sdan] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677335] sd 2:0:28:0: [sdab] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677338] sd 2:0:50:0: [sdax] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677342] sd 2:0:23:0: [sdw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677357] sd 2:0:14:0: [sdn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677361] sd 2:0:22:0: [sdv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677367] sd 2:0:44:0: [sdar] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677373] sd 2:0:36:0: [sdaj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677387] sd 2:0:16:0: [sdp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677389] sd 2:0:29:0: [sdac] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677390] sd 2:0:32:0: [sdaf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677393] sd 2:0:13:0: [sdm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677402] sd 2:0:48:0: [sdav] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677403] sd 2:0:17:0: [sdq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677408] sd 2:0:18:0: [sdr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677410] sd 2:0:71:0: [sdbr] Write Protect is off [ 29.677411] sd 2:0:71:0: [sdbr] Mode Sense: f7 00 10 08 [ 29.677417] sd 2:0:52:0: [sdaz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677424] sd 2:0:19:0: [sds] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677438] sd 2:0:21:0: [sdu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677441] sd 2:0:25:0: [sdy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677446] sd 2:0:39:0: [sdam] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677455] sd 2:0:41:0: [sdao] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677467] sd 2:0:47:0: [sdau] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.677480] sd 2:0:11:0: [sdk] Write Protect is off [ 29.677482] sd 2:0:11:0: [sdk] Mode Sense: f7 00 10 08 [ 29.677483] sd 2:0:43:0: [sdaq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.680463] sd 2:0:26:0: [sdz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.680474] sd 2:0:12:0: [sdl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682792] sd 2:0:81:0: [sdcb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.682794] sd 2:0:81:0: [sdcb] 4096-byte physical blocks [ 29.682802] sd 2:0:56:0: [sdbd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682804] sd 2:0:55:0: [sdbc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682810] sd 2:0:60:0: [sdbh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682816] sd 2:0:54:0: [sdbb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682818] sd 2:0:65:0: [sdbl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682824] sd 2:0:58:0: [sdbf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682830] sd 2:0:57:0: [sdbe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682870] sd 2:0:72:0: [sdbs] Write Protect is off [ 29.682884] sd 2:0:72:0: [sdbs] Mode Sense: f7 00 10 08 [ 29.682920] sd 2:0:66:0: [sdbm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682948] sd 2:0:20:0: [sdt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682948] sd 2:0:67:0: [sdbn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682957] sd 2:0:64:0: [sdbk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682959] sd 2:0:38:0: [sdal] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682967] sd 2:0:24:0: [sdx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682970] sd 2:0:53:0: [sdba] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682971] sd 2:0:83:0: [sdcd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.682972] sd 2:0:83:0: [sdcd] 4096-byte physical blocks [ 29.682975] sd 2:0:59:0: [sdbg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.682997] sd 2:0:63:0: [sdbj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.683004] sd 2:0:82:0: [sdcc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.683005] sd 2:0:82:0: [sdcc] 4096-byte physical blocks [ 29.689466] sd 2:0:42:0: [sdap] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.690914] sd 2:0:68:0: [sdbo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.690921] sd 2:0:73:0: [sdbt] Write Protect is off [ 29.690924] sd 2:0:69:0: [sdbp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.690926] sd 2:0:73:0: [sdbt] Mode Sense: f7 00 10 08 [ 29.690973] sd 2:0:84:0: [sdce] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.690975] sd 2:0:84:0: [sdce] 4096-byte physical blocks [ 29.691062] sd 2:0:71:0: [sdbr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.691076] sd 2:0:70:0: [sdbq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.691079] sd 2:0:11:0: [sdk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.691100] sd 2:0:85:0: [sdcf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.691101] sd 2:0:74:0: [sdbu] Write Protect is off [ 29.691103] sd 2:0:85:0: [sdcf] 4096-byte physical blocks [ 29.691104] sd 2:0:74:0: [sdbu] Mode Sense: f7 00 10 08 [ 29.692476] sd 2:0:61:0: [sdbi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.692488] sd 2:0:75:0: [sdbv] Write Protect is off [ 29.692490] sd 2:0:75:0: [sdbv] Mode Sense: f7 00 10 08 [ 29.696477] sd 2:0:92:0: [sdcm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.696479] sd 2:0:92:0: [sdcm] 4096-byte physical blocks [ 29.696526] sd 2:0:88:0: [sdci] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.696527] sd 2:0:88:0: [sdci] 4096-byte physical blocks [ 29.696536] sd 2:0:78:0: [sdby] Write Protect is off [ 29.696538] sd 2:0:78:0: [sdby] Mode Sense: f7 00 10 08 [ 29.696634] sd 2:0:37:0: [sdak] Write Protect is off [ 29.696636] sd 2:0:37:0: [sdak] Mode Sense: f7 00 10 08 [ 29.696838] sd 2:0:93:0: [sdcn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.696840] sd 2:0:93:0: [sdcn] 4096-byte physical blocks [ 29.696947] sd 2:0:89:0: [sdcj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.696949] sd 2:0:89:0: [sdcj] 4096-byte physical blocks [ 29.697017] sd 2:0:72:0: [sdbs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.697363] sd 2:0:76:0: [sdbw] Write Protect is off [ 29.697365] sd 2:0:76:0: [sdbw] Mode Sense: f7 00 10 08 [ 29.697413] sd 2:0:90:0: [sdck] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.697415] sd 2:0:90:0: [sdck] 4096-byte physical blocks [ 29.697427] sd 2:0:91:0: [sdcl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.697429] sd 2:0:91:0: [sdcl] 4096-byte physical blocks [ 29.697964] sd 2:0:87:0: [sdch] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.697965] sd 2:0:87:0: [sdch] 4096-byte physical blocks [ 29.697976] sd 2:0:77:0: [sdbx] Write Protect is off [ 29.697978] sd 2:0:77:0: [sdbx] Mode Sense: f7 00 10 08 [ 29.704437] sd 2:0:86:0: [sdcg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.704439] sd 2:0:86:0: [sdcg] 4096-byte physical blocks [ 29.704595] sd 2:0:73:0: [sdbt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.704851] sd 2:0:79:0: [sdbz] Write Protect is off [ 29.704852] sd 2:0:79:0: [sdbz] Mode Sense: f7 00 10 08 [ 29.706517] sd 2:0:74:0: [sdbu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.706788] sd 2:0:94:0: [sdco] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.706789] sd 2:0:94:0: [sdco] 4096-byte physical blocks [ 29.706796] sd 2:0:95:0: [sdcp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.706797] sd 2:0:95:0: [sdcp] 4096-byte physical blocks [ 29.710149] sd 2:0:76:0: [sdbw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.710230] sd 2:0:78:0: [sdby] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.710732] sd 2:0:77:0: [sdbx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.711306] sd 2:0:96:0: [sdcq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.711308] sd 2:0:96:0: [sdcq] 4096-byte physical blocks [ 29.711464] sd 2:0:37:0: [sdak] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.718442] sd 2:0:81:0: [sdcb] Write Protect is off [ 29.718443] sd 2:0:83:0: [sdcd] Write Protect is off [ 29.718444] sd 2:0:81:0: [sdcb] Mode Sense: f7 00 10 08 [ 29.718445] sd 2:0:83:0: [sdcd] Mode Sense: f7 00 10 08 [ 29.718500] sd 2:0:79:0: [sdbz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.718607] sd 2:0:80:0: [sdca] Write Protect is off [ 29.718609] sd 2:0:80:0: [sdca] Mode Sense: f7 00 10 08 [ 29.718672] sd 2:0:82:0: [sdcc] Write Protect is off [ 29.718673] sd 2:0:82:0: [sdcc] Mode Sense: f7 00 10 08 [ 29.718705] sd 2:0:97:0: [sdcr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.718707] sd 2:0:97:0: [sdcr] 4096-byte physical blocks [ 29.724175] sd 2:0:100:0: [sdcu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724177] sd 2:0:100:0: [sdcu] 4096-byte physical blocks [ 29.724184] sd 2:0:75:0: [sdbv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.724193] sd 2:0:85:0: [sdcf] Write Protect is off [ 29.724195] sd 2:0:85:0: [sdcf] Mode Sense: f7 00 10 08 [ 29.724200] sd 2:0:107:0: [sddb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724202] sd 2:0:84:0: [sdce] Write Protect is off [ 29.724204] sd 2:0:107:0: [sddb] 4096-byte physical blocks [ 29.724205] sd 2:0:84:0: [sdce] Mode Sense: f7 00 10 08 [ 29.724209] sd 2:0:103:0: [sdcx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724211] sd 2:0:103:0: [sdcx] 4096-byte physical blocks [ 29.724223] sd 2:0:99:0: [sdct] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724225] sd 2:0:99:0: [sdct] 4096-byte physical blocks [ 29.724242] sd 2:0:102:0: [sdcw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724244] sd 2:0:102:0: [sdcw] 4096-byte physical blocks [ 29.724251] sd 2:0:109:0: [sddd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724252] sd 2:0:109:0: [sddd] 4096-byte physical blocks [ 29.724275] sd 2:0:104:0: [sdcy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724276] sd 2:0:104:0: [sdcy] 4096-byte physical blocks [ 29.724279] sd 2:0:105:0: [sdcz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724280] sd 2:0:106:0: [sdda] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724282] sd 2:0:106:0: [sdda] 4096-byte physical blocks [ 29.724285] sd 2:0:105:0: [sdcz] 4096-byte physical blocks [ 29.724292] sd 2:0:108:0: [sddc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724294] sd 2:0:108:0: [sddc] 4096-byte physical blocks [ 29.724591] sd 2:0:101:0: [sdcv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724593] sd 2:0:101:0: [sdcv] 4096-byte physical blocks [ 29.724604] sd 2:0:98:0: [sdcs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.724605] sd 2:0:98:0: [sdcs] 4096-byte physical blocks [ 29.732545] sd 2:0:91:0: [sdcl] Write Protect is off [ 29.732547] sd 2:0:91:0: [sdcl] Mode Sense: f7 00 10 08 [ 29.732557] sd 2:0:83:0: [sdcd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.732604] sd 2:0:90:0: [sdck] Write Protect is off [ 29.732606] sd 2:0:90:0: [sdck] Mode Sense: f7 00 10 08 [ 29.732655] sd 2:0:93:0: [sdcn] Write Protect is off [ 29.732657] sd 2:0:93:0: [sdcn] Mode Sense: f7 00 10 08 [ 29.732738] sd 2:0:82:0: [sdcc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.732744] sd 2:0:80:0: [sdca] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.732754] sd 2:0:110:0: [sdde] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.732755] sd 2:0:110:0: [sdde] 4096-byte physical blocks [ 29.733086] sd 2:0:111:0: [sddf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.733088] sd 2:0:111:0: [sddf] 4096-byte physical blocks [ 29.738431] sd 2:0:84:0: [sdce] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.738469] sd 2:0:85:0: [sdcf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.738505] sd 2:0:94:0: [sdco] Write Protect is off [ 29.738506] sd 2:0:94:0: [sdco] Mode Sense: f7 00 10 08 [ 29.738509] sd 2:0:86:0: [sdcg] Write Protect is off [ 29.738511] sd 2:0:86:0: [sdcg] Mode Sense: f7 00 10 08 [ 29.738517] sd 2:0:95:0: [sdcp] Write Protect is off [ 29.738519] sd 2:0:95:0: [sdcp] Mode Sense: f7 00 10 08 [ 29.738530] sd 2:0:92:0: [sdcm] Write Protect is off [ 29.738531] sd 2:0:92:0: [sdcm] Mode Sense: f7 00 10 08 [ 29.745464] sd 2:0:87:0: [sdch] Write Protect is off [ 29.745465] sd 2:0:87:0: [sdch] Mode Sense: f7 00 10 08 [ 29.799976] sd 2:0:124:0: [sddr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.799978] sd 2:0:124:0: [sddr] 4096-byte physical blocks [ 29.813427] sd 2:0:90:0: [sdck] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.813446] sd 2:0:126:0: [sddt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.813451] sd 2:0:126:0: [sddt] 4096-byte physical blocks [ 29.813524] sd 2:0:89:0: [sdcj] Write Protect is off [ 29.813526] sd 2:0:125:0: [sdds] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.813528] sd 2:0:89:0: [sdcj] Mode Sense: f7 00 10 08 [ 29.813530] sd 2:0:125:0: [sdds] 4096-byte physical blocks [ 29.813645] sd 2:0:91:0: [sdcl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.813652] sd 2:0:88:0: [sdci] Write Protect is off [ 29.813654] sd 2:0:88:0: [sdci] Mode Sense: f7 00 10 08 [ 29.813717] sd 2:0:114:0: [sddi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.813719] sd 2:0:114:0: [sddi] 4096-byte physical blocks [ 29.813815] sd 2:0:96:0: [sdcq] Write Protect is off [ 29.813816] sd 2:0:96:0: [sdcq] Mode Sense: f7 00 10 08 [ 29.821847] sd 2:0:119:0: [sddn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.821849] sd 2:0:119:0: [sddn] 4096-byte physical blocks [ 29.821874] sd 2:0:94:0: [sdco] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.821962] sd 2:0:117:0: [sddl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.821964] sd 2:0:117:0: [sddl] 4096-byte physical blocks [ 29.821980] sd 2:0:81:0: [sdcb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.821985] sd 2:0:87:0: [sdch] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.821991] sd 2:0:121:0: [sddp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.821992] sd 2:0:121:0: [sddp] 4096-byte physical blocks [ 29.822015] sd 2:0:115:0: [sddj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.822017] sd 2:0:115:0: [sddj] 4096-byte physical blocks [ 29.822030] sd 2:0:95:0: [sdcp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.822042] sd 2:0:116:0: [sddk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.822043] sd 2:0:116:0: [sddk] 4096-byte physical blocks [ 29.822057] sd 2:0:120:0: [sddo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.822059] sd 2:0:120:0: [sddo] 4096-byte physical blocks [ 29.822070] sd 2:0:118:0: [sddm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.822072] sd 2:0:118:0: [sddm] 4096-byte physical blocks [ 29.822105] sd 2:0:127:0: [sddu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.822106] sd 2:0:127:0: [sddu] 4096-byte physical blocks [ 29.822113] sd 2:0:86:0: [sdcg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.822114] sd 2:0:92:0: [sdcm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.822130] sd 2:0:93:0: [sdcn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.822136] sd 2:0:97:0: [sdcr] Write Protect is off [ 29.822137] sd 2:0:97:0: [sdcr] Mode Sense: f7 00 10 08 [ 29.827542] sd 2:0:129:0: [sddw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.827544] sd 2:0:129:0: [sddw] 4096-byte physical blocks [ 29.827574] sd 2:0:106:0: [sdda] Write Protect is off [ 29.827577] sd 2:0:106:0: [sdda] Mode Sense: f7 00 10 08 [ 29.827586] sd 2:0:103:0: [sdcx] Write Protect is off [ 29.827587] sd 2:0:103:0: [sdcx] Mode Sense: f7 00 10 08 [ 29.827597] sd 2:0:109:0: [sddd] Write Protect is off [ 29.827598] sd 2:0:104:0: [sdcy] Write Protect is off [ 29.827600] sd 2:0:109:0: [sddd] Mode Sense: f7 00 10 08 [ 29.827602] sd 2:0:104:0: [sdcy] Mode Sense: f7 00 10 08 [ 29.827616] sd 2:0:99:0: [sdct] Write Protect is off [ 29.827618] sd 2:0:99:0: [sdct] Mode Sense: f7 00 10 08 [ 29.827643] sd 2:0:102:0: [sdcw] Write Protect is off [ 29.827645] sd 2:0:102:0: [sdcw] Mode Sense: f7 00 10 08 [ 29.827708] sd 2:0:108:0: [sddc] Write Protect is off [ 29.827709] sd 2:0:108:0: [sddc] Mode Sense: f7 00 10 08 [ 29.827736] sd 2:0:96:0: [sdcq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.827772] sd 2:0:105:0: [sdcz] Write Protect is off [ 29.827774] sd 2:0:107:0: [sddb] Write Protect is off [ 29.827776] sd 2:0:105:0: [sdcz] Mode Sense: f7 00 10 08 [ 29.827778] sd 2:0:107:0: [sddb] Mode Sense: f7 00 10 08 [ 29.827786] sd 2:0:98:0: [sdcs] Write Protect is off [ 29.827787] sd 2:0:98:0: [sdcs] Mode Sense: f7 00 10 08 [ 29.833189] sd 2:0:131:0: [sddy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.833191] sd 2:0:131:0: [sddy] 4096-byte physical blocks [ 29.833196] sd 2:0:128:0: [sddv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.833198] sd 2:0:128:0: [sddv] 4096-byte physical blocks [ 29.833346] sd 2:0:97:0: [sdcr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.841538] sd 2:0:110:0: [sdde] Write Protect is off [ 29.841540] sd 2:0:110:0: [sdde] Mode Sense: f7 00 10 08 [ 29.841555] sd 2:0:106:0: [sdda] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.841563] sd 2:0:108:0: [sddc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.841590] sd 2:0:124:0: [sddr] Write Protect is off [ 29.841591] sd 2:0:124:0: [sddr] Mode Sense: f7 00 10 08 [ 29.841593] sd 2:0:104:0: [sdcy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.841595] sd 2:0:109:0: [sddd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.841688] sd 2:0:113:0: [sddh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.841690] sd 2:0:113:0: [sddh] 4096-byte physical blocks [ 29.841700] sd 2:0:111:0: [sddf] Write Protect is off [ 29.841701] sd 2:0:112:0: [sddg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.841703] sd 2:0:112:0: [sddg] 4096-byte physical blocks [ 29.841704] sd 2:0:111:0: [sddf] Mode Sense: f7 00 10 08 [ 29.841737] sd 2:0:102:0: [sdcw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.841774] sd 2:0:100:0: [sdcu] Write Protect is off [ 29.841776] sd 2:0:100:0: [sdcu] Mode Sense: f7 00 10 08 [ 29.841779] sd 2:0:122:0: [sddq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.841780] sd 2:0:122:0: [sddq] 4096-byte physical blocks [ 29.841795] sd 2:0:105:0: [sdcz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.841826] sd 2:0:107:0: [sddb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.842022] sd 2:0:103:0: [sdcx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.842033] sd 2:0:132:0: [sddz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.842035] sd 2:0:132:0: [sddz] 4096-byte physical blocks [ 29.847389] sd 2:0:125:0: [sdds] Write Protect is off [ 29.847391] sd 2:0:125:0: [sdds] Mode Sense: f7 00 10 08 [ 29.847502] sd 2:0:133:0: [sdea] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.847503] sd 2:0:133:0: [sdea] 4096-byte physical blocks [ 29.847513] sd 2:0:114:0: [sddi] Write Protect is off [ 29.847514] sd 2:0:126:0: [sddt] Write Protect is off [ 29.847516] sd 2:0:126:0: [sddt] Mode Sense: f7 00 10 08 [ 29.847516] sd 2:0:114:0: [sddi] Mode Sense: f7 00 10 08 [ 29.847561] sd 2:0:98:0: [sdcs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.847568] sd 2:0:101:0: [sdcv] Write Protect is off [ 29.847570] sd 2:0:101:0: [sdcv] Mode Sense: f7 00 10 08 [ 29.855643] sd 2:0:124:0: [sddr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.855657] sd 2:0:127:0: [sddu] Write Protect is off [ 29.855659] sd 2:0:127:0: [sddu] Mode Sense: f7 00 10 08 [ 29.855664] sd 2:0:119:0: [sddn] Write Protect is off [ 29.855666] sd 2:0:119:0: [sddn] Mode Sense: f7 00 10 08 [ 29.855715] sd 2:0:130:0: [sddx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.855717] sd 2:0:130:0: [sddx] 4096-byte physical blocks [ 29.855720] sd 2:0:121:0: [sddp] Write Protect is off [ 29.855722] sd 2:0:121:0: [sddp] Mode Sense: f7 00 10 08 [ 29.855788] sd 2:0:115:0: [sddj] Write Protect is off [ 29.855789] sd 2:0:115:0: [sddj] Mode Sense: f7 00 10 08 [ 29.855807] sd 2:0:111:0: [sddf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.855813] sd 2:0:120:0: [sddo] Write Protect is off [ 29.855815] sd 2:0:120:0: [sddo] Mode Sense: f7 00 10 08 [ 29.855818] sd 2:0:99:0: [sdct] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.855896] sd 2:0:134:0: [sdeb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.855898] sd 2:0:134:0: [sdeb] 4096-byte physical blocks [ 29.861329] sd 2:0:126:0: [sddt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.861334] sd 2:0:118:0: [sddm] Write Protect is off [ 29.861336] sd 2:0:118:0: [sddm] Mode Sense: f7 00 10 08 [ 29.861375] sd 2:0:117:0: [sddl] Write Protect is off [ 29.861377] sd 2:0:117:0: [sddl] Mode Sense: f7 00 10 08 [ 29.861393] sd 2:0:116:0: [sddk] Write Protect is off [ 29.861394] sd 2:0:116:0: [sddk] Mode Sense: f7 00 10 08 [ 29.861637] sd 2:0:110:0: [sdde] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.861642] sd 2:0:129:0: [sddw] Write Protect is off [ 29.861644] sd 2:0:129:0: [sddw] Mode Sense: f7 00 10 08 [ 29.862556] sd 2:0:125:0: [sdds] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.869754] sd 2:0:127:0: [sddu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.869761] sd 2:0:128:0: [sddv] Write Protect is off [ 29.869763] sd 2:0:128:0: [sddv] Mode Sense: f7 00 10 08 [ 29.869766] sd 2:0:131:0: [sddy] Write Protect is off [ 29.869768] sd 2:0:131:0: [sddy] Mode Sense: f7 00 10 08 [ 29.869809] sd 2:0:114:0: [sddi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.869862] sd 2:0:121:0: [sddp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.869909] sd 2:0:89:0: [sdcj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.870037] sd 2:0:120:0: [sddo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.870107] sd 2:0:136:0: [sded] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.870109] sd 2:0:136:0: [sded] 4096-byte physical blocks [ 29.871366] sd 2:0:119:0: [sddn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.875653] sd 2:0:135:0: [sdec] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.875655] sd 2:0:135:0: [sdec] 4096-byte physical blocks [ 29.875687] sd 2:0:139:0: [sdeg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.875689] sd 2:0:139:0: [sdeg] 4096-byte physical blocks [ 29.875848] sd 2:0:118:0: [sddm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.875973] sd 2:0:122:0: [sddq] Write Protect is off [ 29.875975] sd 2:0:122:0: [sddq] Mode Sense: f7 00 10 08 [ 29.876098] sd 2:0:132:0: [sddz] Write Protect is off [ 29.876100] sd 2:0:132:0: [sddz] Mode Sense: f7 00 10 08 [ 29.876117] sd 2:0:129:0: [sddw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.876227] sd 2:0:88:0: [sdci] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.876240] sd 2:0:115:0: [sddj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.884300] sd 2:0:128:0: [sddv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.884343] sd 2:0:133:0: [sdea] Write Protect is off [ 29.884345] sd 2:0:133:0: [sdea] Mode Sense: f7 00 10 08 [ 29.884377] sd 2:0:131:0: [sddy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.884702] sd 2:0:117:0: [sddl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.884817] sd 2:0:113:0: [sddh] Write Protect is off [ 29.884819] sd 2:0:113:0: [sddh] Mode Sense: f7 00 10 08 [ 29.885595] sd 2:0:112:0: [sddg] Write Protect is off [ 29.885597] sd 2:0:112:0: [sddg] Mode Sense: f7 00 10 08 [ 29.885609] sd 2:0:137:0: [sdee] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.885611] sd 2:0:137:0: [sdee] 4096-byte physical blocks [ 29.888485] sd 2:0:116:0: [sddk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.890499] sd 2:0:140:0: [sdeh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.890501] sd 2:0:140:0: [sdeh] 4096-byte physical blocks [ 29.890536] sd 2:0:134:0: [sdeb] Write Protect is off [ 29.890538] sd 2:0:134:0: [sdeb] Mode Sense: f7 00 10 08 [ 29.890551] sd 2:0:130:0: [sddx] Write Protect is off [ 29.890552] sd 2:0:130:0: [sddx] Mode Sense: f7 00 10 08 [ 29.890580] sd 2:0:100:0: [sdcu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.890709] sd 2:0:132:0: [sddz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.891340] sd 2:0:138:0: [sdef] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.891342] sd 2:0:138:0: [sdef] 4096-byte physical blocks [ 29.893233] sd 2:0:122:0: [sddq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.899465] sd 2:0:133:0: [sdea] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.899515] sd 2:0:101:0: [sdcv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.899532] sd 2:0:141:0: [sdei] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.899534] sd 2:0:141:0: [sdei] 4096-byte physical blocks [ 29.900169] sd 2:0:112:0: [sddg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.905710] sd 2:0:142:0: [sdej] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.905712] sd 2:0:142:0: [sdej] 4096-byte physical blocks [ 29.905751] sd 2:0:134:0: [sdeb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.905760] sd 2:0:143:0: [sdek] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.905763] sd 2:0:143:0: [sdek] 4096-byte physical blocks [ 29.905775] sd 2:0:144:0: [sdel] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.905777] sd 2:0:144:0: [sdel] 4096-byte physical blocks [ 29.905796] sd 2:0:130:0: [sddx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.906178] sd 2:0:146:0: [sden] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.906180] sd 2:0:146:0: [sden] 4096-byte physical blocks [ 29.906498] sd 2:0:145:0: [sdem] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.906500] sd 2:0:145:0: [sdem] 4096-byte physical blocks [ 29.906581] sd 2:0:136:0: [sded] Write Protect is off [ 29.906582] sd 2:0:136:0: [sded] Mode Sense: f7 00 10 08 [ 29.914549] sd 2:0:139:0: [sdeg] Write Protect is off [ 29.914551] sd 2:0:139:0: [sdeg] Mode Sense: f7 00 10 08 [ 29.918470] sd 2:0:135:0: [sdec] Write Protect is off [ 29.918472] sd 2:0:135:0: [sdec] Mode Sense: f7 00 10 08 [ 29.923723] sd 2:0:151:0: [sdes] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923725] sd 2:0:151:0: [sdes] 4096-byte physical blocks [ 29.923736] sd 2:0:152:0: [sdet] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923738] sd 2:0:152:0: [sdet] 4096-byte physical blocks [ 29.923756] sd 2:0:153:0: [sdeu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923757] sd 2:0:153:0: [sdeu] 4096-byte physical blocks [ 29.923759] sd 2:0:154:0: [sdev] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923764] sd 2:0:155:0: [sdew] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923765] sd 2:0:154:0: [sdev] 4096-byte physical blocks [ 29.923766] sd 2:0:155:0: [sdew] 4096-byte physical blocks [ 29.923780] sd 2:0:156:0: [sdex] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923782] sd 2:0:156:0: [sdex] 4096-byte physical blocks [ 29.923884] sd 2:0:157:0: [sdey] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923886] sd 2:0:157:0: [sdey] 4096-byte physical blocks [ 29.923899] sd 2:0:158:0: [sdez] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923901] sd 2:0:158:0: [sdez] 4096-byte physical blocks [ 29.923908] sd 2:0:150:0: [sder] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.923909] sd 2:0:150:0: [sder] 4096-byte physical blocks [ 29.924887] sd 2:0:136:0: [sded] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.925554] sd 2:0:149:0: [sdeq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.925557] sd 2:0:149:0: [sdeq] 4096-byte physical blocks [ 29.928165] sd 2:0:113:0: [sddh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.929838] sd 2:0:163:0: [sdfe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.929840] sd 2:0:163:0: [sdfe] 4096-byte physical blocks [ 29.929858] sd 2:0:162:0: [sdfd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.929859] sd 2:0:162:0: [sdfd] 4096-byte physical blocks [ 29.929863] sd 2:0:165:0: [sdfg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.929865] sd 2:0:165:0: [sdfg] 4096-byte physical blocks [ 29.929874] sd 2:0:140:0: [sdeh] Write Protect is off [ 29.929876] sd 2:0:140:0: [sdeh] Mode Sense: f7 00 10 08 [ 29.929952] sd 2:0:164:0: [sdff] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.929954] sd 2:0:164:0: [sdff] 4096-byte physical blocks [ 29.930078] sd 2:0:139:0: [sdeg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.930111] sd 2:0:137:0: [sdee] Write Protect is off [ 29.930113] sd 2:0:137:0: [sdee] Mode Sense: f7 00 10 08 [ 29.930176] sd 2:0:159:0: [sdfa] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930178] sd 2:0:168:0: [sdfj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930179] sd 2:0:159:0: [sdfa] 4096-byte physical blocks [ 29.930179] sd 2:0:168:0: [sdfj] 4096-byte physical blocks [ 29.930183] sd 2:0:167:0: [sdfi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930184] sd 2:0:167:0: [sdfi] 4096-byte physical blocks [ 29.930196] sd 2:0:169:0: [sdfk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930197] sd 2:0:169:0: [sdfk] 4096-byte physical blocks [ 29.930201] sd 2:0:174:0: [sdfp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930202] sd 2:0:174:0: [sdfp] 4096-byte physical blocks [ 29.930217] sd 2:0:176:0: [sdfr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930219] sd 2:0:176:0: [sdfr] 4096-byte physical blocks [ 29.930220] sd 2:0:170:0: [sdfl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930222] sd 2:0:170:0: [sdfl] 4096-byte physical blocks [ 29.930229] sd 2:0:175:0: [sdfq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930230] sd 2:0:175:0: [sdfq] 4096-byte physical blocks [ 29.930242] sd 2:0:173:0: [sdfo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930243] sd 2:0:173:0: [sdfo] 4096-byte physical blocks [ 29.930248] sd 2:0:177:0: [sdfs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930250] sd 2:0:177:0: [sdfs] 4096-byte physical blocks [ 29.930304] sd 2:0:187:0: [sdgb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930305] sd 2:0:187:0: [sdgb] 4096-byte physical blocks [ 29.930312] sd 2:0:191:0: [sdgf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930313] sd 2:0:191:0: [sdgf] 4096-byte physical blocks [ 29.930331] sd 2:0:192:0: [sdgg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930332] sd 2:0:192:0: [sdgg] 4096-byte physical blocks [ 29.930341] sd 2:0:193:0: [sdgh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930343] sd 2:0:193:0: [sdgh] 4096-byte physical blocks [ 29.930345] sd 2:0:194:0: [sdgi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930346] sd 2:0:194:0: [sdgi] 4096-byte physical blocks [ 29.930465] sd 2:0:186:0: [sdga] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930466] sd 2:0:186:0: [sdga] 4096-byte physical blocks [ 29.930474] sd 2:0:196:0: [sdgk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930475] sd 2:0:183:0: [sdfy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930476] sd 2:0:196:0: [sdgk] 4096-byte physical blocks [ 29.930477] sd 2:0:183:0: [sdfy] 4096-byte physical blocks [ 29.930485] sd 2:0:188:0: [sdgc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930486] sd 2:0:188:0: [sdgc] 4096-byte physical blocks [ 29.930488] sd 2:0:185:0: [sdfz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930491] sd 2:0:185:0: [sdfz] 4096-byte physical blocks [ 29.930542] sd 2:0:182:0: [sdfx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930544] sd 2:0:182:0: [sdfx] 4096-byte physical blocks [ 29.930757] sd 2:0:135:0: [sdec] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.930770] sd 2:0:179:0: [sdfu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930771] sd 2:0:179:0: [sdfu] 4096-byte physical blocks [ 29.930778] sd 2:0:147:0: [sdeo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930779] sd 2:0:166:0: [sdfh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.930780] sd 2:0:147:0: [sdeo] 4096-byte physical blocks [ 29.930781] sd 2:0:166:0: [sdfh] 4096-byte physical blocks [ 29.931189] sd 2:0:190:0: [sdge] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.931191] sd 2:0:190:0: [sdge] 4096-byte physical blocks [ 29.931200] sd 2:0:189:0: [sdgd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.931201] sd 2:0:189:0: [sdgd] 4096-byte physical blocks [ 29.933468] sd 2:0:195:0: [sdgj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.933469] sd 2:0:195:0: [sdgj] 4096-byte physical blocks [ 29.933924] sd 2:0:178:0: [sdft] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.933925] sd 2:0:178:0: [sdft] 4096-byte physical blocks [ 29.938871] sd 2:0:141:0: [sdei] Write Protect is off [ 29.938873] sd 2:0:141:0: [sdei] Mode Sense: f7 00 10 08 [ 29.938876] sd 2:0:197:0: [sdgl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.938878] sd 2:0:197:0: [sdgl] 4096-byte physical blocks [ 29.938973] sd 2:0:171:0: [sdfm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.938974] sd 2:0:171:0: [sdfm] 4096-byte physical blocks [ 29.939140] sd 2:0:212:0: [sdha] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939142] sd 2:0:212:0: [sdha] 4096-byte physical blocks [ 29.939154] sd 2:0:215:0: [sdhd] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939156] sd 2:0:224:0: [sdhm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939158] sd 2:0:215:0: [sdhd] 4096-byte physical blocks [ 29.939159] sd 2:0:224:0: [sdhm] 4096-byte physical blocks [ 29.939168] sd 2:0:216:0: [sdhe] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939170] sd 2:0:216:0: [sdhe] 4096-byte physical blocks [ 29.939188] sd 2:0:213:0: [sdhb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939189] sd 2:0:213:0: [sdhb] 4096-byte physical blocks [ 29.939193] sd 2:0:228:0: [sdhq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939194] sd 2:0:228:0: [sdhq] 4096-byte physical blocks [ 29.939200] sd 2:0:225:0: [sdhn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939201] sd 2:0:214:0: [sdhc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939202] sd 2:0:225:0: [sdhn] 4096-byte physical blocks [ 29.939206] sd 2:0:214:0: [sdhc] 4096-byte physical blocks [ 29.939217] sd 2:0:231:0: [sdht] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939219] sd 2:0:231:0: [sdht] 4096-byte physical blocks [ 29.939224] sd 2:0:223:0: [sdhl] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939226] sd 2:0:223:0: [sdhl] 4096-byte physical blocks [ 29.939230] sd 2:0:217:0: [sdhf] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939231] sd 2:0:217:0: [sdhf] 4096-byte physical blocks [ 29.939234] sd 2:0:227:0: [sdhp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939235] sd 2:0:227:0: [sdhp] 4096-byte physical blocks [ 29.939241] sd 2:0:218:0: [sdhg] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939243] sd 2:0:218:0: [sdhg] 4096-byte physical blocks [ 29.939253] sd 2:0:230:0: [sdhs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939255] sd 2:0:230:0: [sdhs] 4096-byte physical blocks [ 29.939259] sd 2:0:219:0: [sdhh] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939260] sd 2:0:219:0: [sdhh] 4096-byte physical blocks [ 29.939267] sd 2:0:201:0: [sdgp] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939269] sd 2:0:201:0: [sdgp] 4096-byte physical blocks [ 29.939271] sd 2:0:226:0: [sdho] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939272] sd 2:0:226:0: [sdho] 4096-byte physical blocks [ 29.939283] sd 2:0:203:0: [sdgr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939285] sd 2:0:203:0: [sdgr] 4096-byte physical blocks [ 29.939287] sd 2:0:206:0: [sdgu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939288] sd 2:0:206:0: [sdgu] 4096-byte physical blocks [ 29.939294] sd 2:0:208:0: [sdgw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939296] sd 2:0:208:0: [sdgw] 4096-byte physical blocks [ 29.939400] sd 2:0:205:0: [sdgt] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939401] sd 2:0:205:0: [sdgt] 4096-byte physical blocks [ 29.939411] sd 2:0:161:0: [sdfc] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939412] sd 2:0:161:0: [sdfc] 4096-byte physical blocks [ 29.939419] sd 2:0:229:0: [sdhr] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939420] sd 2:0:229:0: [sdhr] 4096-byte physical blocks [ 29.939424] sd 2:0:160:0: [sdfb] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939425] sd 2:0:160:0: [sdfb] 4096-byte physical blocks [ 29.939465] sd 2:0:204:0: [sdgs] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939466] sd 2:0:204:0: [sdgs] 4096-byte physical blocks [ 29.939507] sd 2:0:199:0: [sdgn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939508] sd 2:0:199:0: [sdgn] 4096-byte physical blocks [ 29.939596] sd 2:0:180:0: [sdfv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939598] sd 2:0:180:0: [sdfv] 4096-byte physical blocks [ 29.939627] sd 2:0:148:0: [sdep] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.939629] sd 2:0:148:0: [sdep] 4096-byte physical blocks [ 29.941256] sd 2:0:138:0: [sdef] Write Protect is off [ 29.941258] sd 2:0:138:0: [sdef] Mode Sense: f7 00 10 08 [ 29.941266] sd 2:0:202:0: [sdgq] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.941267] sd 2:0:202:0: [sdgq] 4096-byte physical blocks [ 29.941276] sd 2:0:200:0: [sdgo] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.941277] sd 2:0:200:0: [sdgo] 4096-byte physical blocks [ 29.945346] sd 2:0:234:0: [sdhw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.945348] sd 2:0:234:0: [sdhw] 4096-byte physical blocks [ 29.945373] sd 2:0:140:0: [sdeh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.945375] sd 2:0:142:0: [sdej] Write Protect is off [ 29.945377] sd 2:0:142:0: [sdej] Mode Sense: f7 00 10 08 [ 29.945397] sd 2:0:143:0: [sdek] Write Protect is off [ 29.945399] sd 2:0:143:0: [sdek] Mode Sense: f7 00 10 08 [ 29.945403] sd 2:0:144:0: [sdel] Write Protect is off [ 29.945405] sd 2:0:144:0: [sdel] Mode Sense: f7 00 10 08 [ 29.945423] sd 2:0:145:0: [sdem] Write Protect is off [ 29.945425] sd 2:0:145:0: [sdem] Mode Sense: f7 00 10 08 [ 29.945536] sd 2:0:146:0: [sden] Write Protect is off [ 29.945538] sd 2:0:146:0: [sden] Mode Sense: f7 00 10 08 [ 29.945900] sd 2:0:221:0: [sdhj] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.945902] sd 2:0:221:0: [sdhj] 4096-byte physical blocks [ 29.945939] sd 2:0:209:0: [sdgx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.945941] sd 2:0:209:0: [sdgx] 4096-byte physical blocks [ 29.945951] sd 2:0:236:0: [sdhy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.945953] sd 2:0:236:0: [sdhy] 4096-byte physical blocks [ 29.945982] sd 2:0:172:0: [sdfn] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.945983] sd 2:0:172:0: [sdfn] 4096-byte physical blocks [ 29.946043] sd 2:0:235:0: [sdhx] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.946045] sd 2:0:235:0: [sdhx] 4096-byte physical blocks [ 29.946237] sd 2:0:3:0: [sdc] Attached SCSI disk [ 29.951725] sd 2:0:207:0: [sdgv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.951727] sd 2:0:207:0: [sdgv] 4096-byte physical blocks [ 29.951832] sd 2:0:141:0: [sdei] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.952348] sd 2:0:63:0: [sdbj] Attached SCSI disk [ 29.952358] sd 2:0:222:0: [sdhk] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.952360] sd 2:0:222:0: [sdhk] 4096-byte physical blocks [ 29.952448] sd 2:0:181:0: [sdfw] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.952457] sd 2:0:181:0: [sdfw] 4096-byte physical blocks [ 29.952556] sd 2:0:233:0: [sdhv] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.952557] sd 2:0:233:0: [sdhv] 4096-byte physical blocks [ 29.958482] sd 2:0:198:0: [sdgm] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.958484] sd 2:0:198:0: [sdgm] 4096-byte physical blocks [ 29.960989] sd 2:0:142:0: [sdej] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.961005] sd 2:0:156:0: [sdex] Write Protect is off [ 29.961007] sd 2:0:156:0: [sdex] Mode Sense: f7 00 10 08 [ 29.961008] sd 2:0:143:0: [sdek] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.961017] sd 2:0:158:0: [sdez] Write Protect is off [ 29.961019] sd 2:0:144:0: [sdel] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.961021] sd 2:0:158:0: [sdez] Mode Sense: f7 00 10 08 [ 29.961025] sd 2:0:151:0: [sdes] Write Protect is off [ 29.961028] sd 2:0:151:0: [sdes] Mode Sense: f7 00 10 08 [ 29.961029] sd 2:0:155:0: [sdew] Write Protect is off [ 29.961031] sd 2:0:155:0: [sdew] Mode Sense: f7 00 10 08 [ 29.961032] sd 2:0:145:0: [sdem] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.961039] sd 2:0:157:0: [sdey] Write Protect is off [ 29.961041] sd 2:0:154:0: [sdev] Write Protect is off [ 29.961042] sd 2:0:152:0: [sdet] Write Protect is off [ 29.961044] sd 2:0:157:0: [sdey] Mode Sense: f7 00 10 08 [ 29.961046] sd 2:0:154:0: [sdev] Mode Sense: f7 00 10 08 [ 29.961047] sd 2:0:152:0: [sdet] Mode Sense: f7 00 10 08 [ 29.961093] sd 2:0:146:0: [sden] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.961254] sd 2:0:153:0: [sdeu] Write Protect is off [ 29.961256] sd 2:0:153:0: [sdeu] Mode Sense: f7 00 10 08 [ 29.961405] sd 2:0:220:0: [sdhi] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961407] sd 2:0:220:0: [sdhi] 4096-byte physical blocks [ 29.961410] sd 2:0:240:0: [sdic] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961412] sd 2:0:240:0: [sdic] 4096-byte physical blocks [ 29.961421] sd 2:0:239:0: [sdib] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961422] sd 2:0:241:0: [sdid] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961423] sd 2:0:239:0: [sdib] 4096-byte physical blocks [ 29.961424] sd 2:0:241:0: [sdid] 4096-byte physical blocks [ 29.961430] sd 2:0:137:0: [sdee] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.961516] sd 2:0:232:0: [sdhu] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961517] sd 2:0:232:0: [sdhu] 4096-byte physical blocks [ 29.961523] sd 2:0:210:0: [sdgy] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961524] sd 2:0:210:0: [sdgy] 4096-byte physical blocks [ 29.961667] sd 2:0:242:0: [sdie] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961670] sd 2:0:242:0: [sdie] 4096-byte physical blocks [ 29.961701] sd 2:0:64:0: [sdbk] Attached SCSI disk [ 29.961779] sd 2:0:237:0: [sdhz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961780] sd 2:0:237:0: [sdhz] 4096-byte physical blocks [ 29.961827] sd 2:0:238:0: [sdia] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.961828] sd 2:0:238:0: [sdia] 4096-byte physical blocks [ 29.967390] sd 2:0:243:0: [sdif] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.967392] sd 2:0:243:0: [sdif] 4096-byte physical blocks [ 29.967410] sd 2:0:165:0: [sdfg] Write Protect is off [ 29.967412] sd 2:0:165:0: [sdfg] Mode Sense: f7 00 10 08 [ 29.967416] sd 2:0:163:0: [sdfe] Write Protect is off [ 29.967418] sd 2:0:163:0: [sdfe] Mode Sense: f7 00 10 08 [ 29.967446] sd 2:0:164:0: [sdff] Write Protect is off [ 29.967447] sd 2:0:164:0: [sdff] Mode Sense: f7 00 10 08 [ 29.967473] sd 2:0:162:0: [sdfd] Write Protect is off [ 29.967474] sd 2:0:167:0: [sdfi] Write Protect is off [ 29.967476] sd 2:0:162:0: [sdfd] Mode Sense: f7 00 10 08 [ 29.967478] sd 2:0:167:0: [sdfi] Mode Sense: f7 00 10 08 [ 29.967479] sd 2:0:169:0: [sdfk] Write Protect is off [ 29.967481] sd 2:0:169:0: [sdfk] Mode Sense: f7 00 10 08 [ 29.967482] sd 2:0:174:0: [sdfp] Write Protect is off [ 29.967483] sd 2:0:174:0: [sdfp] Mode Sense: f7 00 10 08 [ 29.967485] sd 2:0:168:0: [sdfj] Write Protect is off [ 29.967487] sd 2:0:168:0: [sdfj] Mode Sense: f7 00 10 08 [ 29.967494] sd 2:0:191:0: [sdgf] Write Protect is off [ 29.967495] sd 2:0:191:0: [sdgf] Mode Sense: f7 00 10 08 [ 29.967502] sd 2:0:176:0: [sdfr] Write Protect is off [ 29.967504] sd 2:0:187:0: [sdgb] Write Protect is off [ 29.967505] sd 2:0:176:0: [sdfr] Mode Sense: f7 00 10 08 [ 29.967507] sd 2:0:187:0: [sdgb] Mode Sense: f7 00 10 08 [ 29.967536] sd 2:0:177:0: [sdfs] Write Protect is off [ 29.967538] sd 2:0:177:0: [sdfs] Mode Sense: f7 00 10 08 [ 29.967545] sd 2:0:175:0: [sdfq] Write Protect is off [ 29.967546] sd 2:0:175:0: [sdfq] Mode Sense: f7 00 10 08 [ 29.967563] sd 2:0:194:0: [sdgi] Write Protect is off [ 29.967565] sd 2:0:194:0: [sdgi] Mode Sense: f7 00 10 08 [ 29.967579] sd 2:0:193:0: [sdgh] Write Protect is off [ 29.967580] sd 2:0:193:0: [sdgh] Mode Sense: f7 00 10 08 [ 29.967630] sd 2:0:159:0: [sdfa] Write Protect is off [ 29.967631] sd 2:0:159:0: [sdfa] Mode Sense: f7 00 10 08 [ 29.967693] sd 2:0:183:0: [sdfy] Write Protect is off [ 29.967694] sd 2:0:183:0: [sdfy] Mode Sense: f7 00 10 08 [ 29.967703] sd 2:0:170:0: [sdfl] Write Protect is off [ 29.967705] sd 2:0:170:0: [sdfl] Mode Sense: f7 00 10 08 [ 29.967722] sd 2:0:147:0: [sdeo] Write Protect is off [ 29.967723] sd 2:0:147:0: [sdeo] Mode Sense: f7 00 10 08 [ 29.967831] sd 2:0:166:0: [sdfh] Write Protect is off [ 29.967832] sd 2:0:166:0: [sdfh] Mode Sense: f7 00 10 08 [ 29.967837] sd 2:0:173:0: [sdfo] Write Protect is off [ 29.967839] sd 2:0:173:0: [sdfo] Mode Sense: f7 00 10 08 [ 29.967885] sd 2:0:244:0: [sdig] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.967886] sd 2:0:186:0: [sdga] Write Protect is off [ 29.967888] sd 2:0:244:0: [sdig] 4096-byte physical blocks [ 29.967889] sd 2:0:186:0: [sdga] Mode Sense: f7 00 10 08 [ 29.967914] sd 2:0:190:0: [sdge] Write Protect is off [ 29.967915] sd 2:0:190:0: [sdge] Mode Sense: f7 00 10 08 [ 29.967971] sd 2:0:196:0: [sdgk] Write Protect is off [ 29.967973] sd 2:0:196:0: [sdgk] Mode Sense: f7 00 10 08 [ 29.968018] sd 2:0:182:0: [sdfx] Write Protect is off [ 29.968019] sd 2:0:182:0: [sdfx] Mode Sense: f7 00 10 08 [ 29.968030] sd 2:0:189:0: [sdgd] Write Protect is off [ 29.968032] sd 2:0:189:0: [sdgd] Mode Sense: f7 00 10 08 [ 29.968084] sd 2:0:185:0: [sdfz] Write Protect is off [ 29.968086] sd 2:0:185:0: [sdfz] Mode Sense: f7 00 10 08 [ 29.968099] sd 2:0:192:0: [sdgg] Write Protect is off [ 29.968101] sd 2:0:192:0: [sdgg] Mode Sense: f7 00 10 08 [ 29.968109] sd 2:0:149:0: [sdeq] Write Protect is off [ 29.968111] sd 2:0:149:0: [sdeq] Mode Sense: f7 00 10 08 [ 29.968155] sd 2:0:23:0: [sdw] Attached SCSI disk [ 29.968162] sd 2:0:58:0: [sdbf] Attached SCSI disk [ 29.968204] sd 2:0:41:0: [sdao] Attached SCSI disk [ 29.968288] sd 2:0:188:0: [sdgc] Write Protect is off [ 29.968290] sd 2:0:188:0: [sdgc] Mode Sense: f7 00 10 08 [ 29.968371] sd 2:0:195:0: [sdgj] Write Protect is off [ 29.968373] sd 2:0:195:0: [sdgj] Mode Sense: f7 00 10 08 [ 29.968473] sd 2:0:12:0: [sdl] Attached SCSI disk [ 29.968488] sd 2:0:179:0: [sdfu] Write Protect is off [ 29.968490] sd 2:0:179:0: [sdfu] Mode Sense: f7 00 10 08 [ 29.968620] sd 2:0:138:0: [sdef] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.968823] sd 2:0:178:0: [sdft] Write Protect is off [ 29.968825] sd 2:0:178:0: [sdft] Mode Sense: f7 00 10 08 [ 29.976752] sd 2:0:197:0: [sdgl] Write Protect is off [ 29.976753] sd 2:0:197:0: [sdgl] Mode Sense: f7 00 10 08 [ 29.976802] sd 2:0:231:0: [sdht] Write Protect is off [ 29.976804] sd 2:0:231:0: [sdht] Mode Sense: f7 00 10 08 [ 29.976840] sd 2:0:227:0: [sdhp] Write Protect is off [ 29.976842] sd 2:0:227:0: [sdhp] Mode Sense: f7 00 10 08 [ 29.976855] sd 2:0:156:0: [sdex] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.976864] sd 2:0:158:0: [sdez] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.976870] sd 2:0:157:0: [sdey] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.976880] sd 2:0:214:0: [sdhc] Write Protect is off [ 29.976881] sd 2:0:214:0: [sdhc] Mode Sense: f7 00 10 08 [ 29.976884] sd 2:0:151:0: [sdes] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.976889] sd 2:0:160:0: [sdfb] Write Protect is off [ 29.976891] sd 2:0:160:0: [sdfb] Mode Sense: f7 00 10 08 [ 29.976892] sd 2:0:216:0: [sdhe] Write Protect is off [ 29.976894] sd 2:0:216:0: [sdhe] Mode Sense: f7 00 10 08 [ 29.976900] sd 2:0:213:0: [sdhb] Write Protect is off [ 29.976901] sd 2:0:213:0: [sdhb] Mode Sense: f7 00 10 08 [ 29.976904] sd 2:0:154:0: [sdev] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.976953] sd 2:0:224:0: [sdhm] Write Protect is off [ 29.976955] sd 2:0:224:0: [sdhm] Mode Sense: f7 00 10 08 [ 29.976966] sd 2:0:226:0: [sdho] Write Protect is off [ 29.976967] sd 2:0:226:0: [sdho] Mode Sense: f7 00 10 08 [ 29.976972] sd 2:0:161:0: [sdfc] Write Protect is off [ 29.976973] sd 2:0:161:0: [sdfc] Mode Sense: f7 00 10 08 [ 29.976995] sd 2:0:225:0: [sdhn] Write Protect is off [ 29.976997] sd 2:0:225:0: [sdhn] Mode Sense: f7 00 10 08 [ 29.977007] sd 2:0:180:0: [sdfv] Write Protect is off [ 29.977008] sd 2:0:180:0: [sdfv] Mode Sense: f7 00 10 08 [ 29.977016] sd 2:0:230:0: [sdhs] Write Protect is off [ 29.977016] sd 2:0:228:0: [sdhq] Write Protect is off [ 29.977018] sd 2:0:230:0: [sdhs] Mode Sense: f7 00 10 08 [ 29.977018] sd 2:0:228:0: [sdhq] Mode Sense: f7 00 10 08 [ 29.977056] sd 2:0:155:0: [sdew] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.977070] sd 2:0:153:0: [sdeu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.977079] sd 2:0:152:0: [sdet] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.977082] sd 2:0:229:0: [sdhr] Write Protect is off [ 29.977083] sd 2:0:229:0: [sdhr] Mode Sense: f7 00 10 08 [ 29.977086] sd 2:0:217:0: [sdhf] Write Protect is off [ 29.977088] sd 2:0:217:0: [sdhf] Mode Sense: f7 00 10 08 [ 29.977108] sd 2:0:219:0: [sdhh] Write Protect is off [ 29.977110] sd 2:0:219:0: [sdhh] Mode Sense: f7 00 10 08 [ 29.977123] sd 2:0:218:0: [sdhg] Write Protect is off [ 29.977124] sd 2:0:218:0: [sdhg] Mode Sense: f7 00 10 08 [ 29.977149] sd 2:0:215:0: [sdhd] Write Protect is off [ 29.977150] sd 2:0:215:0: [sdhd] Mode Sense: f7 00 10 08 [ 29.977173] sd 2:0:148:0: [sdep] Write Protect is off [ 29.977175] sd 2:0:148:0: [sdep] Mode Sense: f7 00 10 08 [ 29.977177] sd 2:0:212:0: [sdha] Write Protect is off [ 29.977179] sd 2:0:212:0: [sdha] Mode Sense: f7 00 10 08 [ 29.977207] sd 2:0:223:0: [sdhl] Write Protect is off [ 29.977208] sd 2:0:223:0: [sdhl] Mode Sense: f7 00 10 08 [ 29.977242] sd 2:0:208:0: [sdgw] Write Protect is off [ 29.977243] sd 2:0:208:0: [sdgw] Mode Sense: f7 00 10 08 [ 29.977310] sd 2:0:205:0: [sdgt] Write Protect is off [ 29.977311] sd 2:0:205:0: [sdgt] Mode Sense: f7 00 10 08 [ 29.977380] sd 2:0:202:0: [sdgq] Write Protect is off [ 29.977382] sd 2:0:202:0: [sdgq] Mode Sense: f7 00 10 08 [ 29.977461] sd 2:0:203:0: [sdgr] Write Protect is off [ 29.977462] sd 2:0:203:0: [sdgr] Mode Sense: f7 00 10 08 [ 29.977474] sd 2:0:35:0: [sdai] Attached SCSI disk [ 29.977526] sd 2:0:7:0: [sdg] Attached SCSI disk [ 29.977571] sd 2:0:43:0: [sdaq] Attached SCSI disk [ 29.977671] sd 2:0:30:0: [sdad] Attached SCSI disk [ 29.977700] sd 2:0:9:0: [sdi] Attached SCSI disk [ 29.977727] sd 2:0:36:0: [sdaj] Attached SCSI disk [ 29.977738] sd 2:0:204:0: [sdgs] Write Protect is off [ 29.977740] sd 2:0:204:0: [sdgs] Mode Sense: f7 00 10 08 [ 29.977751] sd 2:0:47:0: [sdau] Attached SCSI disk [ 29.977811] sd 2:0:50:0: [sdax] Attached SCSI disk [ 29.978094] sd 2:0:201:0: [sdgp] Write Protect is off [ 29.978099] sd 2:0:201:0: [sdgp] Mode Sense: f7 00 10 08 [ 29.978114] sd 2:0:206:0: [sdgu] Write Protect is off [ 29.978116] sd 2:0:206:0: [sdgu] Mode Sense: f7 00 10 08 [ 29.979466] sd 2:0:48:0: [sdav] Attached SCSI disk [ 29.979474] sd 2:0:200:0: [sdgo] Write Protect is off [ 29.979475] sd 2:0:200:0: [sdgo] Mode Sense: f7 00 10 08 [ 29.983375] sd 2:0:165:0: [sdfg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983376] sd 2:0:234:0: [sdhw] Write Protect is off [ 29.983378] sd 2:0:234:0: [sdhw] Mode Sense: f7 00 10 08 [ 29.983388] sd 2:0:163:0: [sdfe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983397] sd 2:0:164:0: [sdff] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983409] sd 2:0:167:0: [sdfi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983414] sd 2:0:169:0: [sdfk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983426] sd 2:0:168:0: [sdfj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983429] sd 2:0:187:0: [sdgb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983434] sd 2:0:176:0: [sdfr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983439] sd 2:0:177:0: [sdfs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983442] sd 2:0:194:0: [sdgi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983457] sd 2:0:191:0: [sdgf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983462] sd 2:0:175:0: [sdfq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983527] sd 2:0:174:0: [sdfp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983545] sd 2:0:183:0: [sdfy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983558] sd 2:0:170:0: [sdfl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983607] sd 2:0:209:0: [sdgx] Write Protect is off [ 29.983609] sd 2:0:209:0: [sdgx] Mode Sense: f7 00 10 08 [ 29.983638] sd 2:0:166:0: [sdfh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983648] sd 2:0:173:0: [sdfo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983677] sd 2:0:236:0: [sdhy] Write Protect is off [ 29.983678] sd 2:0:236:0: [sdhy] Mode Sense: f7 00 10 08 [ 29.983698] sd 2:0:199:0: [sdgn] Write Protect is off [ 29.983700] sd 2:0:199:0: [sdgn] Mode Sense: f7 00 10 08 [ 29.983749] sd 2:0:182:0: [sdfx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983771] sd 2:0:149:0: [sdeq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983775] sd 2:0:235:0: [sdhx] Write Protect is off [ 29.983777] sd 2:0:235:0: [sdhx] Mode Sense: f7 00 10 08 [ 29.983779] sd 2:0:185:0: [sdfz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983799] sd 2:0:190:0: [sdge] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983810] sd 2:0:193:0: [sdgh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983862] sd 2:0:179:0: [sdfu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.983890] sd 2:0:211:0: [sdgz] 15628053168 512-byte logical blocks: (8.00 TB/7.27 TiB) [ 29.983891] sd 2:0:211:0: [sdgz] 4096-byte physical blocks [ 29.983911] sd 2:0:150:0: [sder] Write Protect is off [ 29.983913] sd 2:0:150:0: [sder] Mode Sense: f7 00 10 08 [ 29.983957] sd 2:0:195:0: [sdgj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.984017] sd 2:0:6:0: [sdf] Attached SCSI disk [ 29.984034] sd 2:0:42:0: [sdap] Attached SCSI disk [ 29.984066] sd 2:0:60:0: [sdbh] Attached SCSI disk [ 29.984081] sd 2:0:66:0: [sdbm] Attached SCSI disk [ 29.984084] sd 2:0:172:0: [sdfn] Write Protect is off [ 29.984085] sd 2:0:172:0: [sdfn] Mode Sense: f7 00 10 08 [ 29.984214] sd 2:0:189:0: [sdgd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.984250] sd 2:0:14:0: [sdn] Attached SCSI disk [ 29.984253] sd 2:0:53:0: [sdba] Attached SCSI disk [ 29.984257] sd 2:0:33:0: [sdag] Attached SCSI disk [ 29.984258] sd 2:0:159:0: [sdfa] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.984266] sd 2:0:192:0: [sdgg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.984297] sd 2:0:31:0: [sdae] Attached SCSI disk [ 29.984329] sd 2:0:34:0: [sdah] Attached SCSI disk [ 29.984335] sd 2:0:17:0: [sdq] Attached SCSI disk [ 29.984370] sd 2:0:188:0: [sdgc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.984384] sd 2:0:25:0: [sdy] Attached SCSI disk [ 29.984421] sd 2:0:5:0: [sde] Attached SCSI disk [ 29.984460] sd 2:0:32:0: [sdaf] Attached SCSI disk [ 29.984475] sd 2:0:24:0: [sdx] Attached SCSI disk [ 29.984486] sd 2:0:178:0: [sdft] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.985472] sd 2:0:54:0: [sdbb] Attached SCSI disk [ 29.985479] sd 2:0:171:0: [sdfm] Write Protect is off [ 29.985480] sd 2:0:171:0: [sdfm] Mode Sense: f7 00 10 08 [ 29.990462] sd 2:0:13:0: [sdm] Attached SCSI disk [ 29.992652] sd 2:0:196:0: [sdgk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992672] sd 2:0:231:0: [sdht] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992675] sd 2:0:207:0: [sdgv] Write Protect is off [ 29.992677] sd 2:0:207:0: [sdgv] Mode Sense: f7 00 10 08 [ 29.992682] sd 2:0:227:0: [sdhp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992691] sd 2:0:18:0: [sdr] Attached SCSI disk [ 29.992700] sd 2:0:214:0: [sdhc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992714] sd 2:0:213:0: [sdhb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992718] sd 2:0:216:0: [sdhe] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992752] sd 2:0:181:0: [sdfw] Write Protect is off [ 29.992755] sd 2:0:181:0: [sdfw] Mode Sense: f7 00 10 08 [ 29.992769] sd 2:0:161:0: [sdfc] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992803] sd 2:0:180:0: [sdfv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992862] sd 2:0:205:0: [sdgt] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992865] sd 2:0:198:0: [sdgm] Write Protect is off [ 29.992867] sd 2:0:198:0: [sdgm] Mode Sense: f7 00 10 08 [ 29.992907] sd 2:0:148:0: [sdep] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992916] sd 2:0:203:0: [sdgr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.992930] sd 2:0:222:0: [sdhk] Write Protect is off [ 29.992933] sd 2:0:222:0: [sdhk] Mode Sense: f7 00 10 08 [ 29.993025] sd 2:0:186:0: [sdga] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993064] sd 2:0:204:0: [sdgs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993099] sd 2:0:221:0: [sdhj] Write Protect is off [ 29.993101] sd 2:0:221:0: [sdhj] Mode Sense: f7 00 10 08 [ 29.993161] sd 2:0:225:0: [sdhn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993171] sd 2:0:217:0: [sdhf] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993178] sd 2:0:219:0: [sdhh] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993184] sd 2:0:215:0: [sdhd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993188] sd 2:0:212:0: [sdha] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993203] sd 2:0:8:0: [sdh] Attached SCSI disk [ 29.993243] sd 2:0:224:0: [sdhm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993249] sd 2:0:49:0: [sdaw] Attached SCSI disk [ 29.993270] sd 2:0:147:0: [sdeo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993274] sd 2:0:228:0: [sdhq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993295] sd 2:0:78:0: [sdby] Attached SCSI disk [ 29.993302] sd 2:0:233:0: [sdhv] Write Protect is off [ 29.993304] sd 2:0:233:0: [sdhv] Mode Sense: f7 00 10 08 [ 29.993332] sd 2:0:229:0: [sdhr] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993375] sd 2:0:67:0: [sdbn] Attached SCSI disk [ 29.993405] sd 2:0:226:0: [sdho] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993424] sd 2:0:29:0: [sdac] Attached SCSI disk [ 29.993442] sd 2:0:162:0: [sdfd] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993502] sd 2:0:59:0: [sdbg] Attached SCSI disk [ 29.993543] sd 2:0:73:0: [sdbt] Attached SCSI disk [ 29.993647] sd 2:0:68:0: [sdbo] Attached SCSI disk [ 29.993674] sd 2:0:218:0: [sdhg] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993678] sd 2:0:46:0: [sdat] Attached SCSI disk [ 29.993692] sd 2:0:230:0: [sdhs] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993716] sd 2:0:201:0: [sdgp] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993719] sd 2:0:200:0: [sdgo] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993721] sd 2:0:206:0: [sdgu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.993739] sd 2:0:57:0: [sdbe] Attached SCSI disk [ 29.999374] sd 2:0:240:0: [sdic] Write Protect is off [ 29.999376] sd 2:0:240:0: [sdic] Mode Sense: f7 00 10 08 [ 29.999379] sd 2:0:239:0: [sdib] Write Protect is off [ 29.999380] sd 2:0:239:0: [sdib] Mode Sense: f7 00 10 08 [ 29.999385] sd 2:0:220:0: [sdhi] Write Protect is off [ 29.999387] sd 2:0:220:0: [sdhi] Mode Sense: f7 00 10 08 [ 29.999425] sd 2:0:234:0: [sdhw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999447] sd 2:0:199:0: [sdgn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999448] sd 2:0:209:0: [sdgx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999459] sd 2:0:210:0: [sdgy] Write Protect is off [ 29.999461] sd 2:0:241:0: [sdid] Write Protect is off [ 29.999462] sd 2:0:210:0: [sdgy] Mode Sense: f7 00 10 08 [ 29.999463] sd 2:0:241:0: [sdid] Mode Sense: f7 00 10 08 [ 29.999543] sd 2:0:197:0: [sdgl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999576] sd 2:0:238:0: [sdia] Write Protect is off [ 29.999577] sd 2:0:150:0: [sder] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999579] sd 2:0:238:0: [sdia] Mode Sense: f7 00 10 08 [ 29.999589] sd 2:0:202:0: [sdgq] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999595] sd 2:0:236:0: [sdhy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999598] sd 2:0:242:0: [sdie] Write Protect is off [ 29.999601] sd 2:0:242:0: [sdie] Mode Sense: f7 00 10 08 [ 29.999706] sd 2:0:172:0: [sdfn] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 29.999824] sd 2:0:79:0: [sdbz] Attached SCSI disk [ 29.999829] sd 2:0:26:0: [sdz] Attached SCSI disk [ 29.999840] sd 2:0:74:0: [sdbu] Attached SCSI disk [ 29.999863] sd 2:0:237:0: [sdhz] Write Protect is off [ 29.999865] sd 2:0:237:0: [sdhz] Mode Sense: f7 00 10 08 [ 29.999914] sd 2:0:21:0: [sdu] Attached SCSI disk [ 30.000018] sd 2:0:11:0: [sdk] Attached SCSI disk [ 30.000158] sd 2:0:208:0: [sdgw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.000496] sd 2:0:10:0: [sdj] Attached SCSI disk [ 30.000677] sd 2:0:160:0: [sdfb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.001483] sd 2:0:19:0: [sds] Attached SCSI disk [ 30.007463] sd 2:0:232:0: [sdhu] Write Protect is off [ 30.007465] sd 2:0:232:0: [sdhu] Mode Sense: f7 00 10 08 [ 30.008415] sd 2:0:207:0: [sdgv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.008431] sd 2:0:243:0: [sdif] Write Protect is off [ 30.008432] sd 2:0:243:0: [sdif] Mode Sense: f7 00 10 08 [ 30.008468] sd 2:0:244:0: [sdig] Write Protect is off [ 30.008470] sd 2:0:244:0: [sdig] Mode Sense: f7 00 10 08 [ 30.008499] sd 2:0:198:0: [sdgm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.008572] sd 2:0:233:0: [sdhv] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.008678] sd 2:0:221:0: [sdhj] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.008957] sd 2:0:22:0: [sdv] Attached SCSI disk [ 30.008960] sd 2:0:72:0: [sdbs] Attached SCSI disk [ 30.009095] sd 2:0:222:0: [sdhk] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.009157] sd 2:0:171:0: [sdfm] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.009303] sd 2:0:69:0: [sdbp] Attached SCSI disk [ 30.009990] sd 2:0:38:0: [sdal] Attached SCSI disk [ 30.017297] sd 2:0:240:0: [sdic] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017320] sd 2:0:220:0: [sdhi] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017395] sd 2:0:241:0: [sdid] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017400] sd 2:0:210:0: [sdgy] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017426] sd 2:0:239:0: [sdib] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017528] sd 2:0:181:0: [sdfw] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017554] sd 2:0:238:0: [sdia] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017577] sd 2:0:237:0: [sdhz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.017916] sd 2:0:75:0: [sdbv] Attached SCSI disk [ 30.018118] sd 2:0:56:0: [sdbd] Attached SCSI disk [ 30.018138] sd 2:0:70:0: [sdbq] Attached SCSI disk [ 30.018729] sd 2:0:223:0: [sdhl] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.019418] sd 2:0:84:0: [sdce] Attached SCSI disk [ 30.023473] sd 2:0:232:0: [sdhu] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.023734] sd 2:0:243:0: [sdif] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.023955] sd 2:0:242:0: [sdie] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.024006] sd 2:0:37:0: [sdak] Attached SCSI disk [ 30.024061] sd 2:0:20:0: [sdt] Attached SCSI disk [ 30.029987] sd 2:0:211:0: [sdgz] Write Protect is off [ 30.029989] sd 2:0:211:0: [sdgz] Mode Sense: f7 00 10 08 [ 30.030028] sd 2:0:44:0: [sdar] Attached SCSI disk [ 30.030051] sd 2:0:244:0: [sdig] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.030139] sd 2:0:51:0: [sday] Attached SCSI disk [ 30.030244] sd 2:0:4:0: [sdd] Attached SCSI disk [ 30.030296] sd 2:0:61:0: [sdbi] Attached SCSI disk [ 30.030343] sd 2:0:45:0: [sdas] Attached SCSI disk [ 30.031454] sd 2:0:235:0: [sdhx] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.038932] sd 2:0:40:0: [sdan] Attached SCSI disk [ 30.039020] sd 2:0:39:0: [sdam] Attached SCSI disk [ 30.039026] sd 2:0:80:0: [sdca] Attached SCSI disk [ 30.039115] sd 2:0:15:0: [sdo] Attached SCSI disk [ 30.039124] sd 2:0:65:0: [sdbl] Attached SCSI disk [ 30.045410] sd 2:0:211:0: [sdgz] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 30.045476] sd 2:0:55:0: [sdbc] Attached SCSI disk [ 30.045791] sd 2:0:71:0: [sdbr] Attached SCSI disk [ 30.054475] sd 2:0:28:0: [sdab] Attached SCSI disk [ 30.054839] sd 2:0:81:0: [sdcb] Attached SCSI disk [ 30.061026] sd 2:0:16:0: [sdp] Attached SCSI disk [ 30.061027] sd 2:0:87:0: [sdch] Attached SCSI disk [ 30.069675] sd 2:0:102:0: [sdcw] Attached SCSI disk [ 30.069820] sd 2:0:90:0: [sdck] Attached SCSI disk [ 30.069827] sd 2:0:85:0: [sdcf] Attached SCSI disk [ 30.069830] sd 2:0:95:0: [sdcp] Attached SCSI disk [ 30.069908] sd 2:0:76:0: [sdbw] Attached SCSI disk [ 30.078490] sd 2:0:83:0: [sdcd] Attached SCSI disk [ 30.078591] sd 2:0:91:0: [sdcl] Attached SCSI disk [ 30.078595] sd 2:0:86:0: [sdcg] Attached SCSI disk [ 30.078649] sd 2:0:106:0: [sdda] Attached SCSI disk [ 30.078653] sd 2:0:27:0: [sdaa] Attached SCSI disk [ 30.078694] sd 2:0:52:0: [sdaz] Attached SCSI disk [ 30.078730] sd 2:0:82:0: [sdcc] Attached SCSI disk [ 30.084531] sd 2:0:97:0: [sdcr] Attached SCSI disk [ 30.084538] sd 2:0:103:0: [sdcx] Attached SCSI disk [ 30.084606] sd 2:0:96:0: [sdcq] Attached SCSI disk [ 30.084653] sd 2:0:94:0: [sdco] Attached SCSI disk [ 30.084729] sd 2:0:108:0: [sddc] Attached SCSI disk [ 30.090660] sd 2:0:124:0: [sddr] Attached SCSI disk [ 30.090715] sd 2:0:111:0: [sddf] Attached SCSI disk [ 30.090738] sd 2:0:92:0: [sdcm] Attached SCSI disk [ 30.090921] sd 2:0:119:0: [sddn] Attached SCSI disk [ 30.099251] sd 2:0:110:0: [sdde] Attached SCSI disk [ 30.099290] sd 2:0:104:0: [sdcy] Attached SCSI disk [ 30.099347] sd 2:0:93:0: [sdcn] Attached SCSI disk [ 30.099368] sd 2:0:109:0: [sddd] Attached SCSI disk [ 30.105317] sd 2:0:121:0: [sddp] Attached SCSI disk [ 30.105442] sd 2:0:114:0: [sddi] Attached SCSI disk [ 30.105585] sd 2:0:99:0: [sdct] Attached SCSI disk [ 30.114242] sd 2:0:107:0: [sddb] Attached SCSI disk [ 30.114500] sd 2:0:120:0: [sddo] Attached SCSI disk [ 30.114605] sd 2:0:125:0: [sdds] Attached SCSI disk [ 30.115471] sd 2:0:127:0: [sddu] Attached SCSI disk [ 30.123020] sd 2:0:128:0: [sddv] Attached SCSI disk [ 30.123323] sd 2:0:77:0: [sdbx] Attached SCSI disk [ 30.123339] sd 2:0:115:0: [sddj] Attached SCSI disk [ 30.129346] sd 2:0:101:0: [sdcv] Attached SCSI disk [ 30.129503] sd 2:0:133:0: [sdea] Attached SCSI disk [ 30.129611] sd 2:0:98:0: [sdcs] Attached SCSI disk [ 30.133146] sd 2:0:105:0: [sdcz] Attached SCSI disk [ 30.138334] sd 2:0:112:0: [sddg] Attached SCSI disk [ 30.138437] sd 2:0:122:0: [sddq] Attached SCSI disk [ 30.140119] sd 2:0:89:0: [sdcj] Attached SCSI disk [ 30.144531] sd 2:0:134:0: [sdeb] Attached SCSI disk [ 30.144550] sd 2:0:118:0: [sddm] Attached SCSI disk [ 30.145680] sd 2:0:117:0: [sddl] Attached SCSI disk [ 30.150521] sd 2:0:100:0: [sdcu] Attached SCSI disk [ 30.159341] sd 2:0:136:0: [sded] Attached SCSI disk [ 30.159352] sd 2:0:116:0: [sddk] Attached SCSI disk [ 30.159353] sd 2:0:140:0: [sdeh] Attached SCSI disk [ 30.160310] sd 2:0:129:0: [sddw] Attached SCSI disk [ 30.165447] sd 2:0:131:0: [sddy] Attached SCSI disk [ 30.165553] sd 2:0:135:0: [sdec] Attached SCSI disk [ 30.165782] sd 2:0:132:0: [sddz] Attached SCSI disk [ 30.174069] sd 2:0:113:0: [sddh] Attached SCSI disk [ 30.174081] sd 2:0:146:0: [sden] Attached SCSI disk [ 30.174082] sd 2:0:141:0: [sdei] Attached SCSI disk [ 30.174107] sd 2:0:126:0: [sddt] Attached SCSI disk [ 30.174263] sd 2:0:139:0: [sdeg] Attached SCSI disk [ 30.174331] sd 2:0:88:0: [sdci] Attached SCSI disk [ 30.180147] sd 2:0:143:0: [sdek] Attached SCSI disk [ 30.180284] sd 2:0:165:0: [sdfg] Attached SCSI disk [ 30.180473] sd 2:0:130:0: [sddx] Attached SCSI disk [ 30.182137] sd 2:0:159:0: [sdfa] Attached SCSI disk [ 30.188692] sd 2:0:163:0: [sdfe] Attached SCSI disk [ 30.188707] sd 2:0:225:0: [sdhn] Attached SCSI disk [ 30.188810] sd 2:0:155:0: [sdew] Attached SCSI disk [ 30.188839] sd 2:0:153:0: [sdeu] Attached SCSI disk [ 30.188843] sd 2:0:231:0: [sdht] Attached SCSI disk [ 30.188958] sd 2:0:151:0: [sdes] Attached SCSI disk [ 30.188962] sd 2:0:152:0: [sdet] Attached SCSI disk [ 30.188967] sd 2:0:156:0: [sdex] Attached SCSI disk [ 30.189023] sd 2:0:169:0: [sdfk] Attached SCSI disk [ 30.197292] sd 2:0:188:0: [sdgc] Attached SCSI disk [ 30.197313] sd 2:0:144:0: [sdel] Attached SCSI disk [ 30.197663] sd 2:0:145:0: [sdem] Attached SCSI disk [ 30.197669] sd 2:0:236:0: [sdhy] Attached SCSI disk [ 30.203354] sd 2:0:201:0: [sdgp] Attached SCSI disk [ 30.203424] sd 2:0:208:0: [sdgw] Attached SCSI disk [ 30.203649] sd 2:0:176:0: [sdfr] Attached SCSI disk [ 30.203650] sd 2:0:179:0: [sdfu] Attached SCSI disk [ 30.203670] sd 2:0:175:0: [sdfq] Attached SCSI disk [ 30.203673] sd 2:0:142:0: [sdej] Attached SCSI disk [ 30.209310] sd 2:0:166:0: [sdfh] Attached SCSI disk [ 30.209398] sd 2:0:158:0: [sdez] Attached SCSI disk [ 30.209442] sd 2:0:189:0: [sdgd] Attached SCSI disk [ 30.209466] sd 2:0:233:0: [sdhv] Attached SCSI disk [ 30.209467] sd 2:0:195:0: [sdgj] Attached SCSI disk [ 30.209471] sd 2:0:197:0: [sdgl] Attached SCSI disk [ 30.209477] sd 2:0:185:0: [sdfz] Attached SCSI disk [ 30.209479] sd 2:0:180:0: [sdfv] Attached SCSI disk [ 30.209481] sd 2:0:200:0: [sdgo] Attached SCSI disk [ 30.209484] sd 2:0:168:0: [sdfj] Attached SCSI disk [ 30.209486] sd 2:0:167:0: [sdfi] Attached SCSI disk [ 30.209490] sd 2:0:209:0: [sdgx] Attached SCSI disk [ 30.209496] sd 2:0:221:0: [sdhj] Attached SCSI disk [ 30.210471] sd 2:0:160:0: [sdfb] Attached SCSI disk [ 30.217929] sd 2:0:218:0: [sdhg] Attached SCSI disk [ 30.217930] sd 2:0:164:0: [sdff] Attached SCSI disk [ 30.217942] sd 2:0:171:0: [sdfm] Attached SCSI disk [ 30.217960] sd 2:0:148:0: [sdep] Attached SCSI disk [ 30.217966] sd 2:0:227:0: [sdhp] Attached SCSI disk [ 30.218172] sd 2:0:240:0: [sdic] Attached SCSI disk [ 30.218180] sd 2:0:177:0: [sdfs] Attached SCSI disk [ 30.218182] sd 2:0:172:0: [sdfn] Attached SCSI disk [ 30.218189] sd 2:0:186:0: [sdga] Attached SCSI disk [ 30.218213] sd 2:0:182:0: [sdfx] Attached SCSI disk [ 30.223902] sd 2:0:212:0: [sdha] Attached SCSI disk [ 30.223903] sd 2:0:216:0: [sdhe] Attached SCSI disk [ 30.223920] sd 2:0:194:0: [sdgi] Attached SCSI disk [ 30.223936] sd 2:0:157:0: [sdey] Attached SCSI disk [ 30.223943] sd 2:0:214:0: [sdhc] Attached SCSI disk [ 30.223982] sd 2:0:229:0: [sdhr] Attached SCSI disk [ 30.224025] sd 2:0:219:0: [sdhh] Attached SCSI disk [ 30.224049] sd 2:0:241:0: [sdid] Attached SCSI disk [ 30.224055] sd 2:0:196:0: [sdgk] Attached SCSI disk [ 30.224069] sd 2:0:170:0: [sdfl] Attached SCSI disk [ 30.232214] sd 2:0:230:0: [sdhs] Attached SCSI disk [ 30.232311] sd 2:0:206:0: [sdgu] Attached SCSI disk [ 30.232319] sd 2:0:242:0: [sdie] Attached SCSI disk [ 30.232320] sd 2:0:149:0: [sdeq] Attached SCSI disk [ 30.232334] sd 2:0:204:0: [sdgs] Attached SCSI disk [ 30.232335] sd 2:0:203:0: [sdgr] Attached SCSI disk [ 30.232372] sd 2:0:224:0: [sdhm] Attached SCSI disk [ 30.232373] sd 2:0:178:0: [sdft] Attached SCSI disk [ 30.232374] sd 2:0:202:0: [sdgq] Attached SCSI disk [ 30.232377] sd 2:0:217:0: [sdhf] Attached SCSI disk [ 30.232472] sd 2:0:226:0: [sdho] Attached SCSI disk [ 30.232492] sd 2:0:207:0: [sdgv] Attached SCSI disk [ 30.238073] sd 2:0:137:0: [sdee] Attached SCSI disk [ 30.238124] sd 2:0:154:0: [sdev] Attached SCSI disk [ 30.238138] sd 2:0:232:0: [sdhu] Attached SCSI disk [ 30.238167] sd 2:0:238:0: [sdia] Attached SCSI disk [ 30.238218] sd 2:0:213:0: [sdhb] Attached SCSI disk [ 30.238245] sd 2:0:220:0: [sdhi] Attached SCSI disk [ 30.246527] sd 2:0:205:0: [sdgt] Attached SCSI disk [ 30.246559] sd 2:0:237:0: [sdhz] Attached SCSI disk [ 30.246610] sd 2:0:239:0: [sdib] Attached SCSI disk [ 30.252237] sd 2:0:228:0: [sdhq] Attached SCSI disk [ 30.252321] sd 2:0:191:0: [sdgf] Attached SCSI disk [ 30.260531] sd 2:0:181:0: [sdfw] Attached SCSI disk [ 30.266145] sd 2:0:174:0: [sdfp] Attached SCSI disk [ 30.266154] sd 2:0:192:0: [sdgg] Attached SCSI disk [ 30.266168] sd 2:0:235:0: [sdhx] Attached SCSI disk [ 30.266186] sd 2:0:193:0: [sdgh] Attached SCSI disk [ 30.266198] sd 2:0:190:0: [sdge] Attached SCSI disk [ 30.274486] sd 2:0:147:0: [sdeo] Attached SCSI disk [ 30.280050] sd 2:0:215:0: [sdhd] Attached SCSI disk [ 30.293593] sd 2:0:138:0: [sdef] Attached SCSI disk [ 30.293666] sd 2:0:243:0: [sdif] Attached SCSI disk [ 30.293696] sd 2:0:150:0: [sder] Attached SCSI disk [ 30.315370] sd 2:0:162:0: [sdfd] Attached SCSI disk [ 30.342302] sd 2:0:187:0: [sdgb] Attached SCSI disk [ 30.342305] sd 2:0:161:0: [sdfc] Attached SCSI disk [ 30.355750] sd 2:0:173:0: [sdfo] Attached SCSI disk [ 30.355763] sd 2:0:210:0: [sdgy] Attached SCSI disk [ 30.382769] sd 2:0:222:0: [sdhk] Attached SCSI disk [ 30.382772] sd 2:0:223:0: [sdhl] Attached SCSI disk [ 30.396246] sd 2:0:198:0: [sdgm] Attached SCSI disk [ 30.409639] sd 2:0:211:0: [sdgz] Attached SCSI disk [ 30.409644] sd 2:0:199:0: [sdgn] Attached SCSI disk [ 30.423022] sd 2:0:183:0: [sdfy] Attached SCSI disk [ 30.436465] sd 2:0:234:0: [sdhw] Attached SCSI disk [ 30.436474] sd 2:0:244:0: [sdig] Attached SCSI disk [ 37.319049] sd 2:0:2:0: [sdb] Write Protect is off [ 37.323858] sd 2:0:2:0: [sdb] Mode Sense: f7 00 10 08 [ 37.324329] sd 2:0:2:0: [sdb] Write cache: enabled, read cache: enabled, supports DPO and FUA [ 37.362852] sd 2:0:2:0: [sdb] Attached SCSI disk [ 37.456658] EXT4-fs (sda2): mounted filesystem with ordered data mode. Opts: (null) [ 37.768540] systemd-journald[358]: Received SIGTERM from PID 1 (systemd). [ 37.807239] SELinux: Disabled at runtime. [ 37.811506] SELinux: Unregistering netfilter hooks [ 37.852525] type=1404 audit(1584563458.341:2): selinux=0 auid=4294967295 ses=4294967295 [ 37.884285] ip_tables: (C) 2000-2006 Netfilter Core Team [ 37.890191] systemd[1]: Inserted module 'ip_tables' [ 38.005194] EXT4-fs (sda2): re-mounted. Opts: (null) [ 38.016223] systemd-journald[5926]: Received request to flush runtime journal from PID 1 [ 38.108152] piix4_smbus 0000:00:14.0: SMBus Host Controller at 0xb00, revision 0 [ 38.116676] piix4_smbus 0000:00:14.0: Using register 0x2e for SMBus port selection [ 38.126890] ACPI Error: No handler for Region [SYSI] (ffff9feca9e79a68) [IPMI] (20130517/evregion-162) [ 38.139096] ACPI Error: Region IPMI (ID=7) has no handler (20130517/exfldio-305) [ 38.150705] ACPI Error: Method parse/execution failed [\_SB_.PMI0._GHL] (Node ffff9feca9e765a0), AE_NOT_EXIST (20130517/psparse-536) [ 38.169640] ACPI Error: Method parse/execution failed [\_SB_.PMI0._PMC] (Node ffff9feca9e76500), AE_NOT_EXIST (20130517/psparse-536) [ 38.188815] ccp 0000:02:00.2: 3 command queues available [ 38.188863] ccp 0000:02:00.2: irq 235 for MSI/MSI-X [ 38.188875] ccp 0000:02:00.2: irq 236 for MSI/MSI-X [ 38.189386] ipmi message handler version 39.2 [ 38.195112] ccp 0000:02:00.2: Queue 2 can access 4 LSB regions [ 38.195115] ccp 0000:02:00.2: Queue 3 can access 4 LSB regions [ 38.195117] ccp 0000:02:00.2: Queue 4 can access 4 LSB regions [ 38.195119] ccp 0000:02:00.2: Queue 0 gets LSB 4 [ 38.195121] ccp 0000:02:00.2: Queue 1 gets LSB 5 [ 38.195122] ccp 0000:02:00.2: Queue 2 gets LSB 6 [ 38.196833] ccp 0000:02:00.2: enabled [ 38.197062] ccp 0000:03:00.1: 5 command queues available [ 38.197101] ccp 0000:03:00.1: irq 238 for MSI/MSI-X [ 38.197140] ccp 0000:03:00.1: Queue 0 can access 7 LSB regions [ 38.197142] ccp 0000:03:00.1: Queue 1 can access 7 LSB regions [ 38.197144] ccp 0000:03:00.1: Queue 2 can access 7 LSB regions [ 38.197146] ccp 0000:03:00.1: Queue 3 can access 7 LSB regions [ 38.197148] ccp 0000:03:00.1: Queue 4 can access 7 LSB regions [ 38.197149] ccp 0000:03:00.1: Queue 0 gets LSB 1 [ 38.197151] ccp 0000:03:00.1: Queue 1 gets LSB 2 [ 38.197152] ccp 0000:03:00.1: Queue 2 gets LSB 3 [ 38.197153] ccp 0000:03:00.1: Queue 3 gets LSB 4 [ 38.197154] ccp 0000:03:00.1: Queue 4 gets LSB 5 [ 38.198361] ccp 0000:03:00.1: enabled [ 38.198596] ccp 0000:41:00.2: 3 command queues available [ 38.198629] ccp 0000:41:00.2: irq 240 for MSI/MSI-X [ 38.198641] ccp 0000:41:00.2: irq 241 for MSI/MSI-X [ 38.198750] ccp 0000:41:00.2: Queue 2 can access 4 LSB regions [ 38.198752] ccp 0000:41:00.2: Queue 3 can access 4 LSB regions [ 38.198754] ccp 0000:41:00.2: Queue 4 can access 4 LSB regions [ 38.198756] ccp 0000:41:00.2: Queue 0 gets LSB 4 [ 38.198757] ccp 0000:41:00.2: Queue 1 gets LSB 5 [ 38.198758] ccp 0000:41:00.2: Queue 2 gets LSB 6 [ 38.199866] ccp 0000:41:00.2: enabled [ 38.200019] ccp 0000:42:00.1: 5 command queues available [ 38.200054] ccp 0000:42:00.1: irq 243 for MSI/MSI-X [ 38.200105] ccp 0000:42:00.1: Queue 0 can access 7 LSB regions [ 38.200107] ccp 0000:42:00.1: Queue 1 can access 7 LSB regions [ 38.200110] ccp 0000:42:00.1: Queue 2 can access 7 LSB regions [ 38.200112] ccp 0000:42:00.1: Queue 3 can access 7 LSB regions [ 38.200114] ccp 0000:42:00.1: Queue 4 can access 7 LSB regions [ 38.200116] ccp 0000:42:00.1: Queue 0 gets LSB 1 [ 38.200118] ccp 0000:42:00.1: Queue 1 gets LSB 2 [ 38.200119] ccp 0000:42:00.1: Queue 2 gets LSB 3 [ 38.200120] ccp 0000:42:00.1: Queue 3 gets LSB 4 [ 38.200121] ccp 0000:42:00.1: Queue 4 gets LSB 5 [ 38.201709] ccp 0000:42:00.1: enabled [ 38.201948] ccp 0000:85:00.2: 3 command queues available [ 38.201987] ccp 0000:85:00.2: irq 245 for MSI/MSI-X [ 38.201998] ccp 0000:85:00.2: irq 246 for MSI/MSI-X [ 38.202089] ccp 0000:85:00.2: Queue 2 can access 4 LSB regions [ 38.202091] ccp 0000:85:00.2: Queue 3 can access 4 LSB regions [ 38.202093] ccp 0000:85:00.2: Queue 4 can access 4 LSB regions [ 38.202095] ccp 0000:85:00.2: Queue 0 gets LSB 4 [ 38.202096] ccp 0000:85:00.2: Queue 1 gets LSB 5 [ 38.202097] ccp 0000:85:00.2: Queue 2 gets LSB 6 [ 38.205556] ccp 0000:85:00.2: enabled [ 38.210158] ccp 0000:86:00.1: 5 command queues available [ 38.210194] ccp 0000:86:00.1: irq 248 for MSI/MSI-X [ 38.210291] ccp 0000:86:00.1: Queue 0 can access 7 LSB regions [ 38.210293] ccp 0000:86:00.1: Queue 1 can access 7 LSB regions [ 38.210296] ccp 0000:86:00.1: Queue 2 can access 7 LSB regions [ 38.210298] ccp 0000:86:00.1: Queue 3 can access 7 LSB regions [ 38.210300] ccp 0000:86:00.1: Queue 4 can access 7 LSB regions [ 38.210301] ccp 0000:86:00.1: Queue 0 gets LSB 1 [ 38.210303] ccp 0000:86:00.1: Queue 1 gets LSB 2 [ 38.210304] ccp 0000:86:00.1: Queue 2 gets LSB 3 [ 38.210305] ccp 0000:86:00.1: Queue 3 gets LSB 4 [ 38.210307] ccp 0000:86:00.1: Queue 4 gets LSB 5 [ 38.212256] ccp 0000:86:00.1: enabled [ 38.212530] ccp 0000:c2:00.2: 3 command queues available [ 38.212570] ccp 0000:c2:00.2: irq 250 for MSI/MSI-X [ 38.212591] ccp 0000:c2:00.2: irq 251 for MSI/MSI-X [ 38.212692] ccp 0000:c2:00.2: Queue 2 can access 4 LSB regions [ 38.212694] ccp 0000:c2:00.2: Queue 3 can access 4 LSB regions [ 38.212696] ccp 0000:c2:00.2: Queue 4 can access 4 LSB regions [ 38.212698] ccp 0000:c2:00.2: Queue 0 gets LSB 4 [ 38.212699] ccp 0000:c2:00.2: Queue 1 gets LSB 5 [ 38.212700] ccp 0000:c2:00.2: Queue 2 gets LSB 6 [ 38.214225] ccp 0000:c2:00.2: enabled [ 38.214386] ccp 0000:c3:00.1: 5 command queues available [ 38.214419] ccp 0000:c3:00.1: irq 253 for MSI/MSI-X [ 38.214485] ccp 0000:c3:00.1: Queue 0 can access 7 LSB regions [ 38.214487] ccp 0000:c3:00.1: Queue 1 can access 7 LSB regions [ 38.214489] ccp 0000:c3:00.1: Queue 2 can access 7 LSB regions [ 38.214491] ccp 0000:c3:00.1: Queue 3 can access 7 LSB regions [ 38.214493] ccp 0000:c3:00.1: Queue 4 can access 7 LSB regions [ 38.214495] ccp 0000:c3:00.1: Queue 0 gets LSB 1 [ 38.214496] ccp 0000:c3:00.1: Queue 1 gets LSB 2 [ 38.214497] ccp 0000:c3:00.1: Queue 2 gets LSB 3 [ 38.214498] ccp 0000:c3:00.1: Queue 3 gets LSB 4 [ 38.214499] ccp 0000:c3:00.1: Queue 4 gets LSB 5 [ 38.216788] ccp 0000:c3:00.1: enabled [ 38.336168] sd 0:2:0:0: Attached scsi generic sg0 type 0 [ 38.337475] scsi 2:0:0:0: Attached scsi generic sg1 type 13 [ 38.338181] scsi 2:0:1:0: Attached scsi generic sg2 type 13 [ 38.340496] sd 2:0:2:0: Attached scsi generic sg3 type 0 [ 38.342245] sd 2:0:3:0: Attached scsi generic sg4 type 0 [ 38.342707] sd 2:0:4:0: Attached scsi generic sg5 type 0 [ 38.343026] sd 2:0:5:0: Attached scsi generic sg6 type 0 [ 38.343329] sd 2:0:6:0: Attached scsi generic sg7 type 0 [ 38.343533] sd 2:0:7:0: Attached scsi generic sg8 type 0 [ 38.343680] sd 2:0:8:0: Attached scsi generic sg9 type 0 [ 38.343851] sd 2:0:9:0: Attached scsi generic sg10 type 0 [ 38.345377] sd 2:0:10:0: Attached scsi generic sg11 type 0 [ 38.347492] sd 2:0:11:0: Attached scsi generic sg12 type 0 [ 38.347854] sd 2:0:12:0: Attached scsi generic sg13 type 0 [ 38.350554] sd 2:0:13:0: Attached scsi generic sg14 type 0 [ 38.352206] sd 2:0:14:0: Attached scsi generic sg15 type 0 [ 38.353000] sd 2:0:15:0: Attached scsi generic sg16 type 0 [ 38.355448] sd 2:0:16:0: Attached scsi generic sg17 type 0 [ 38.355773] sd 2:0:17:0: Attached scsi generic sg18 type 0 [ 38.356243] sd 2:0:18:0: Attached scsi generic sg19 type 0 [ 38.358046] sd 2:0:19:0: Attached scsi generic sg20 type 0 [ 38.358583] sd 2:0:20:0: Attached scsi generic sg21 type 0 [ 38.360025] sd 2:0:21:0: Attached scsi generic sg22 type 0 [ 38.188683] ACPI Exception: AE_NOT_EXIST, Evaluating _PMC (20130517/power_meter-753) [ 38.829174] sd 2:0:22:0: Attached scsi generic sg23 type 0 [ 38.835950] sd 2:0:23:0: Attached scsi generic sg24 type 0 [ 38.842817] sd 2:0:24:0: Attached scsi generic sg25 type 0 [ 38.849883] sd 2:0:25:0: Attached scsi generic sg26 type 0 [ 38.856896] sd 2:0:26:0: Attached scsi generic sg27 type 0 [ 38.857806] ipmi device interface [ 38.868205] sd 2:0:27:0: Attached scsi generic sg28 type 0 [ 38.874832] sd 2:0:28:0: Attached scsi generic sg29 type 0 [ 38.881719] sd 2:0:29:0: Attached scsi generic sg30 type 0 [ 38.890815] sd 2:0:30:0: Attached scsi generic sg31 type 0 [ 38.898175] IPMI System Interface driver [ 38.898392] sd 2:0:31:0: Attached scsi generic sg32 type 0 [ 38.900284] sd 2:0:32:0: Attached scsi generic sg33 type 0 [ 38.914697] ipmi_si dmi-ipmi-si.0: ipmi_platform: probing via SMBIOS [ 38.914882] sd 2:0:33:0: Attached scsi generic sg34 type 0 [ 38.915240] sd 2:0:34:0: Attached scsi generic sg35 type 0 [ 38.915879] sd 2:0:35:0: Attached scsi generic sg36 type 0 [ 38.916661] sd 2:0:36:0: Attached scsi generic sg37 type 0 [ 38.919681] sd 2:0:37:0: Attached scsi generic sg38 type 0 [ 38.920251] sd 2:0:38:0: Attached scsi generic sg39 type 0 [ 38.920905] sd 2:0:39:0: Attached scsi generic sg40 type 0 [ 38.921339] sd 2:0:40:0: Attached scsi generic sg41 type 0 [ 38.921837] sd 2:0:41:0: Attached scsi generic sg42 type 0 [ 38.922327] sd 2:0:42:0: Attached scsi generic sg43 type 0 [ 38.922715] sd 2:0:43:0: Attached scsi generic sg44 type 0 [ 38.989363] ipmi_si: SMBIOS: io 0xca8 regsize 1 spacing 4 irq 10 [ 38.995414] ipmi_si: Adding SMBIOS-specified kcs state machine [ 38.995415] sd 2:0:44:0: Attached scsi generic sg45 type 0 [ 38.995763] sd 2:0:45:0: Attached scsi generic sg46 type 0 [ 38.996312] sd 2:0:46:0: Attached scsi generic sg47 type 0 [ 38.998140] sd 2:0:47:0: Attached scsi generic sg48 type 0 [ 38.998815] sd 2:0:48:0: Attached scsi generic sg49 type 0 [ 39.000488] device-mapper: uevent: version 1.0.3 [ 39.001316] sd 2:0:49:0: Attached scsi generic sg50 type 0 [ 39.003318] sd 2:0:50:0: Attached scsi generic sg51 type 0 [ 39.004663] device-mapper: ioctl: 4.37.1-ioctl (2018-04-03) initialised: dm-devel@redhat.com [ 39.004804] sd 2:0:51:0: Attached scsi generic sg52 type 0 [ 39.005960] sd 2:0:52:0: Attached scsi generic sg53 type 0 [ 39.007245] sd 2:0:53:0: Attached scsi generic sg54 type 0 [ 39.007919] sd 2:0:54:0: Attached scsi generic sg55 type 0 [ 39.008364] sd 2:0:55:0: Attached scsi generic sg56 type 0 [ 39.008638] sd 2:0:56:0: Attached scsi generic sg57 type 0 [ 39.009075] sd 2:0:57:0: Attached scsi generic sg58 type 0 [ 39.009561] sd 2:0:58:0: Attached scsi generic sg59 type 0 [ 39.010184] sd 2:0:59:0: Attached scsi generic sg60 type 0 [ 39.012130] sd 2:0:60:0: Attached scsi generic sg61 type 0 [ 39.012386] sd 2:0:61:0: Attached scsi generic sg62 type 0 [ 39.012537] scsi 2:0:62:0: Attached scsi generic sg63 type 13 [ 39.012751] sd 2:0:63:0: Attached scsi generic sg64 type 0 [ 39.014046] sd 2:0:64:0: Attached scsi generic sg65 type 0 [ 39.014082] sd 2:0:65:0: Attached scsi generic sg66 type 0 [ 39.014118] sd 2:0:66:0: Attached scsi generic sg67 type 0 [ 39.014152] sd 2:0:67:0: Attached scsi generic sg68 type 0 [ 39.014190] sd 2:0:68:0: Attached scsi generic sg69 type 0 [ 39.014230] sd 2:0:69:0: Attached scsi generic sg70 type 0 [ 39.014265] sd 2:0:70:0: Attached scsi generic sg71 type 0 [ 39.014310] sd 2:0:71:0: Attached scsi generic sg72 type 0 [ 39.014361] sd 2:0:72:0: Attached scsi generic sg73 type 0 [ 39.014410] sd 2:0:73:0: Attached scsi generic sg74 type 0 [ 39.014449] sd 2:0:74:0: Attached scsi generic sg75 type 0 [ 39.014484] sd 2:0:75:0: Attached scsi generic sg76 type 0 [ 39.014534] sd 2:0:76:0: Attached scsi generic sg77 type 0 [ 39.014578] sd 2:0:77:0: Attached scsi generic sg78 type 0 [ 39.014645] sd 2:0:78:0: Attached scsi generic sg79 type 0 [ 39.017653] sd 2:0:79:0: Attached scsi generic sg80 type 0 [ 39.019633] sd 2:0:80:0: Attached scsi generic sg81 type 0 [ 39.019874] sd 2:0:81:0: Attached scsi generic sg82 type 0 [ 39.020168] sd 2:0:82:0: Attached scsi generic sg83 type 0 [ 39.021786] sd 2:0:83:0: Attached scsi generic sg84 type 0 [ 39.022121] sd 2:0:84:0: Attached scsi generic sg85 type 0 [ 39.024640] sd 2:0:85:0: Attached scsi generic sg86 type 0 [ 39.024806] sd 2:0:86:0: Attached scsi generic sg87 type 0 [ 39.024992] sd 2:0:87:0: Attached scsi generic sg88 type 0 [ 39.025150] sd 2:0:88:0: Attached scsi generic sg89 type 0 [ 39.025302] sd 2:0:89:0: Attached scsi generic sg90 type 0 [ 39.025425] sd 2:0:90:0: Attached scsi generic sg91 type 0 [ 39.025601] sd 2:0:91:0: Attached scsi generic sg92 type 0 [ 39.029397] sd 2:0:92:0: Attached scsi generic sg93 type 0 [ 39.029744] sd 2:0:93:0: Attached scsi generic sg94 type 0 [ 39.030188] sd 2:0:94:0: Attached scsi generic sg95 type 0 [ 39.033590] sd 2:0:95:0: Attached scsi generic sg96 type 0 [ 39.043627] sd 2:0:96:0: Attached scsi generic sg97 type 0 [ 39.043906] sd 2:0:97:0: Attached scsi generic sg98 type 0 [ 39.044241] sd 2:0:98:0: Attached scsi generic sg99 type 0 [ 39.044636] sd 2:0:99:0: Attached scsi generic sg100 type 0 [ 39.044845] sd 2:0:100:0: Attached scsi generic sg101 type 0 [ 39.045047] sd 2:0:101:0: Attached scsi generic sg102 type 0 [ 39.047574] sd 2:0:102:0: Attached scsi generic sg103 type 0 [ 39.047977] sd 2:0:103:0: Attached scsi generic sg104 type 0 [ 39.050340] sd 2:0:104:0: Attached scsi generic sg105 type 0 [ 39.052662] sd 2:0:105:0: Attached scsi generic sg106 type 0 [ 39.052988] sd 2:0:106:0: Attached scsi generic sg107 type 0 [ 39.053377] sd 2:0:107:0: Attached scsi generic sg108 type 0 [ 39.056827] sd 2:0:108:0: Attached scsi generic sg109 type 0 [ 39.062436] sd 2:0:109:0: Attached scsi generic sg110 type 0 [ 39.064441] sd 2:0:110:0: Attached scsi generic sg111 type 0 [ 39.064887] sd 2:0:111:0: Attached scsi generic sg112 type 0 [ 39.066437] sd 2:0:112:0: Attached scsi generic sg113 type 0 [ 39.067021] sd 2:0:113:0: Attached scsi generic sg114 type 0 [ 39.067591] sd 2:0:114:0: Attached scsi generic sg115 type 0 [ 39.068163] sd 2:0:115:0: Attached scsi generic sg116 type 0 [ 39.068658] sd 2:0:116:0: Attached scsi generic sg117 type 0 [ 39.069337] sd 2:0:117:0: Attached scsi generic sg118 type 0 [ 39.069804] sd 2:0:118:0: Attached scsi generic sg119 type 0 [ 39.070479] sd 2:0:119:0: Attached scsi generic sg120 type 0 [ 39.071020] sd 2:0:120:0: Attached scsi generic sg121 type 0 [ 39.071440] sd 2:0:121:0: Attached scsi generic sg122 type 0 [ 39.071788] sd 2:0:122:0: Attached scsi generic sg123 type 0 [ 39.073337] scsi 2:0:123:0: Attached scsi generic sg124 type 13 [ 39.073782] sd 2:0:124:0: Attached scsi generic sg125 type 0 [ 39.074334] sd 2:0:125:0: Attached scsi generic sg126 type 0 [ 39.074798] sd 2:0:126:0: Attached scsi generic sg127 type 0 [ 39.075413] sd 2:0:127:0: Attached scsi generic sg128 type 0 [ 39.077894] sd 2:0:128:0: Attached scsi generic sg129 type 0 [ 39.078169] sd 2:0:129:0: Attached scsi generic sg130 type 0 [ 39.079355] sd 2:0:130:0: Attached scsi generic sg131 type 0 [ 39.079713] sd 2:0:131:0: Attached scsi generic sg132 type 0 [ 39.079918] sd 2:0:132:0: Attached scsi generic sg133 type 0 [ 39.080156] sd 2:0:133:0: Attached scsi generic sg134 type 0 [ 39.080650] sd 2:0:134:0: Attached scsi generic sg135 type 0 [ 39.081284] sd 2:0:135:0: Attached scsi generic sg136 type 0 [ 39.081748] sd 2:0:136:0: Attached scsi generic sg137 type 0 [ 39.082054] sd 2:0:137:0: Attached scsi generic sg138 type 0 [ 39.082351] sd 2:0:138:0: Attached scsi generic sg139 type 0 [ 39.082633] sd 2:0:139:0: Attached scsi generic sg140 type 0 [ 39.082884] sd 2:0:140:0: Attached scsi generic sg141 type 0 [ 39.085709] sd 2:0:141:0: Attached scsi generic sg142 type 0 [ 39.086111] sd 2:0:142:0: Attached scsi generic sg143 type 0 [ 39.086678] sd 2:0:143:0: Attached scsi generic sg144 type 0 [ 39.087188] sd 2:0:144:0: Attached scsi generic sg145 type 0 [ 39.087543] sd 2:0:145:0: Attached scsi generic sg146 type 0 [ 39.087858] sd 2:0:146:0: Attached scsi generic sg147 type 0 [ 39.088655] sd 2:0:147:0: Attached scsi generic sg148 type 0 [ 39.089041] sd 2:0:148:0: Attached scsi generic sg149 type 0 [ 39.089306] sd 2:0:149:0: Attached scsi generic sg150 type 0 [ 39.089502] sd 2:0:150:0: Attached scsi generic sg151 type 0 [ 39.089857] sd 2:0:151:0: Attached scsi generic sg152 type 0 [ 39.090922] sd 2:0:152:0: Attached scsi generic sg153 type 0 [ 39.091379] sd 2:0:153:0: Attached scsi generic sg154 type 0 [ 39.093576] sd 2:0:154:0: Attached scsi generic sg155 type 0 [ 39.094068] sd 2:0:155:0: Attached scsi generic sg156 type 0 [ 39.094810] sd 2:0:156:0: Attached scsi generic sg157 type 0 [ 39.095254] sd 2:0:157:0: Attached scsi generic sg158 type 0 [ 39.095632] sd 2:0:158:0: Attached scsi generic sg159 type 0 [ 39.096200] sd 2:0:159:0: Attached scsi generic sg160 type 0 [ 39.097703] sd 2:0:160:0: Attached scsi generic sg161 type 0 [ 39.097935] sd 2:0:161:0: Attached scsi generic sg162 type 0 [ 39.098089] sd 2:0:162:0: Attached scsi generic sg163 type 0 [ 39.098294] sd 2:0:163:0: Attached scsi generic sg164 type 0 [ 39.098677] sd 2:0:164:0: Attached scsi generic sg165 type 0 [ 39.099599] sd 2:0:165:0: Attached scsi generic sg166 type 0 [ 39.100328] sd 2:0:166:0: Attached scsi generic sg167 type 0 [ 39.100984] sd 2:0:167:0: Attached scsi generic sg168 type 0 [ 39.103306] sd 2:0:168:0: Attached scsi generic sg169 type 0 [ 39.103804] sd 2:0:169:0: Attached scsi generic sg170 type 0 [ 39.104372] sd 2:0:170:0: Attached scsi generic sg171 type 0 [ 39.104993] sd 2:0:171:0: Attached scsi generic sg172 type 0 [ 39.105587] sd 2:0:172:0: Attached scsi generic sg173 type 0 [ 39.106175] sd 2:0:173:0: Attached scsi generic sg174 type 0 [ 39.107515] sd 2:0:174:0: Attached scsi generic sg175 type 0 [ 39.108134] sd 2:0:175:0: Attached scsi generic sg176 type 0 [ 39.108655] sd 2:0:176:0: Attached scsi generic sg177 type 0 [ 39.109234] sd 2:0:177:0: Attached scsi generic sg178 type 0 [ 39.109530] sd 2:0:178:0: Attached scsi generic sg179 type 0 [ 39.110071] sd 2:0:179:0: Attached scsi generic sg180 type 0 [ 39.110759] sd 2:0:180:0: Attached scsi generic sg181 type 0 [ 39.111478] sd 2:0:181:0: Attached scsi generic sg182 type 0 [ 39.111867] sd 2:0:182:0: Attached scsi generic sg183 type 0 [ 39.112329] sd 2:0:183:0: Attached scsi generic sg184 type 0 [ 39.112563] scsi 2:0:184:0: Attached scsi generic sg185 type 13 [ 39.112955] sd 2:0:185:0: Attached scsi generic sg186 type 0 [ 39.113334] sd 2:0:186:0: Attached scsi generic sg187 type 0 [ 39.114725] sd 2:0:187:0: Attached scsi generic sg188 type 0 [ 39.116558] sd 2:0:188:0: Attached scsi generic sg189 type 0 [ 39.116913] sd 2:0:189:0: Attached scsi generic sg190 type 0 [ 39.117605] sd 2:0:190:0: Attached scsi generic sg191 type 0 [ 39.118230] sd 2:0:191:0: Attached scsi generic sg192 type 0 [ 39.118688] sd 2:0:192:0: Attached scsi generic sg193 type 0 [ 39.119093] sd 2:0:193:0: Attached scsi generic sg194 type 0 [ 39.119603] sd 2:0:194:0: Attached scsi generic sg195 type 0 [ 39.120193] sd 2:0:195:0: Attached scsi generic sg196 type 0 [ 39.120771] sd 2:0:196:0: Attached scsi generic sg197 type 0 [ 39.121374] sd 2:0:197:0: Attached scsi generic sg198 type 0 [ 39.122367] sd 2:0:198:0: Attached scsi generic sg199 type 0 [ 39.122829] sd 2:0:199:0: Attached scsi generic sg200 type 0 [ 39.123218] sd 2:0:200:0: Attached scsi generic sg201 type 0 [ 39.123747] sd 2:0:201:0: Attached scsi generic sg202 type 0 [ 39.124137] sd 2:0:202:0: Attached scsi generic sg203 type 0 [ 39.124637] sd 2:0:203:0: Attached scsi generic sg204 type 0 [ 39.125605] sd 2:0:204:0: Attached scsi generic sg205 type 0 [ 39.125904] sd 2:0:205:0: Attached scsi generic sg206 type 0 [ 39.126910] sd 2:0:206:0: Attached scsi generic sg207 type 0 [ 39.127613] sd 2:0:207:0: Attached scsi generic sg208 type 0 [ 39.128587] sd 2:0:208:0: Attached scsi generic sg209 type 0 [ 39.129167] sd 2:0:209:0: Attached scsi generic sg210 type 0 [ 39.129897] sd 2:0:210:0: Attached scsi generic sg211 type 0 [ 39.130440] sd 2:0:211:0: Attached scsi generic sg212 type 0 [ 39.131124] sd 2:0:212:0: Attached scsi generic sg213 type 0 [ 39.131592] sd 2:0:213:0: Attached scsi generic sg214 type 0 [ 39.131811] sd 2:0:214:0: Attached scsi generic sg215 type 0 [ 39.133237] sd 2:0:215:0: Attached scsi generic sg216 type 0 [ 39.133766] sd 2:0:216:0: Attached scsi generic sg217 type 0 [ 39.134287] sd 2:0:217:0: Attached scsi generic sg218 type 0 [ 39.135025] sd 2:0:218:0: Attached scsi generic sg219 type 0 [ 39.135714] sd 2:0:219:0: Attached scsi generic sg220 type 0 [ 39.136314] sd 2:0:220:0: Attached scsi generic sg221 type 0 [ 39.136818] sd 2:0:221:0: Attached scsi generic sg222 type 0 [ 39.138116] sd 2:0:222:0: Attached scsi generic sg223 type 0 [ 39.138884] sd 2:0:223:0: Attached scsi generic sg224 type 0 [ 39.140110] sd 2:0:224:0: Attached scsi generic sg225 type 0 [ 39.140529] sd 2:0:225:0: Attached scsi generic sg226 type 0 [ 39.141032] sd 2:0:226:0: Attached scsi generic sg227 type 0 [ 39.141605] sd 2:0:227:0: Attached scsi generic sg228 type 0 [ 39.141954] sd 2:0:228:0: Attached scsi generic sg229 type 0 [ 39.142458] sd 2:0:229:0: Attached scsi generic sg230 type 0 [ 39.142998] sd 2:0:230:0: Attached scsi generic sg231 type 0 [ 39.143450] sd 2:0:231:0: Attached scsi generic sg232 type 0 [ 39.143922] sd 2:0:232:0: Attached scsi generic sg233 type 0 [ 39.144375] sd 2:0:233:0: Attached scsi generic sg234 type 0 [ 39.144749] sd 2:0:234:0: Attached scsi generic sg235 type 0 [ 39.145225] sd 2:0:235:0: Attached scsi generic sg236 type 0 [ 39.145723] sd 2:0:236:0: Attached scsi generic sg237 type 0 [ 39.146367] sd 2:0:237:0: Attached scsi generic sg238 type 0 [ 39.147009] sd 2:0:238:0: Attached scsi generic sg239 type 0 [ 39.147488] sd 2:0:239:0: Attached scsi generic sg240 type 0 [ 39.148051] sd 2:0:240:0: Attached scsi generic sg241 type 0 [ 39.148676] sd 2:0:241:0: Attached scsi generic sg242 type 0 [ 39.149363] sd 2:0:242:0: Attached scsi generic sg243 type 0 [ 39.149849] sd 2:0:243:0: Attached scsi generic sg244 type 0 [ 39.150428] sd 2:0:244:0: Attached scsi generic sg245 type 0 [ 39.577060] ipmi_si IPI0001:00: ipmi_platform: probing via ACPI [ 39.577088] ipmi_si IPI0001:00: [io 0x0ca8] regsize 1 spacing 4 irq 10 [ 39.577091] ipmi_si dmi-ipmi-si.0: Removing SMBIOS-specified kcs state machine in favor of ACPI [ 39.577092] ipmi_si: Adding ACPI-specified kcs state machine [ 39.577179] ipmi_si: Trying ACPI-specified kcs state machine at i/o address 0xca8, slave address 0x20, irq 10 [ 39.602597] ipmi_si IPI0001:00: The BMC does not support setting the recv irq bit, compensating, but the BMC needs to be fixed. [ 39.608622] ipmi_si IPI0001:00: Using irq 10 [ 39.631580] ipmi_si IPI0001:00: Found new BMC (man_id: 0x0002a2, prod_id: 0x0100, dev_id: 0x20) [ 39.704526] ipmi_si IPI0001:00: IPMI kcs interface initialized [ 40.848399] device-mapper: multipath service-time: version 0.3.0 loaded [ 46.491482] input: PC Speaker as /devices/platform/pcspkr/input/input2 [ 46.554669] ses 2:0:0:0: Attached Enclosure device [ 46.559560] ses 2:0:1:0: Attached Enclosure device [ 46.564594] ses 2:0:62:0: Attached Enclosure device [ 46.569561] ses 2:0:123:0: Attached Enclosure device [ 46.574605] ses 2:0:184:0: Attached Enclosure device [ 46.580063] cryptd: max_cpu_qlen set to 1000 [ 46.602861] AVX2 version of gcm_enc/dec engaged. [ 46.607485] AES CTR mode by8 optimization enabled [ 46.617597] alg: No test for __gcm-aes-aesni (__driver-gcm-aes-aesni) [ 46.624833] alg: No test for __generic-gcm-aes-aesni (__driver-generic-gcm-aes-aesni) [ 46.708656] kvm: Nested Paging enabled [ 46.723219] MCE: In-kernel MCE decoding enabled. [ 46.732088] AMD64 EDAC driver v3.4.0 [ 46.735698] EDAC amd64: DRAM ECC enabled. [ 46.739718] EDAC amd64: F17h detected (node 0). [ 46.744304] EDAC MC: UMC0 chip selects: [ 46.744307] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.749016] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 46.753731] EDAC amd64: MC: 4: 0MB 5: 0MB [ 46.758447] EDAC amd64: MC: 6: 0MB 7: 0MB [ 46.763167] EDAC MC: UMC1 chip selects: [ 46.763170] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.767884] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 46.772600] EDAC amd64: MC: 4: 0MB 5: 0MB [ 46.777310] EDAC amd64: MC: 6: 0MB 7: 0MB [ 46.782019] EDAC amd64: using x8 syndromes. [ 46.786216] EDAC amd64: MCT channel count: 2 [ 46.798424] EDAC MC0: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:18.3 [ 46.805852] EDAC amd64: DRAM ECC enabled. [ 46.809876] EDAC amd64: F17h detected (node 1). [ 46.814464] EDAC MC: UMC0 chip selects: [ 46.814467] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.819181] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 46.823896] EDAC amd64: MC: 4: 0MB 5: 0MB [ 46.828616] EDAC amd64: MC: 6: 0MB 7: 0MB [ 46.833331] EDAC MC: UMC1 chip selects: [ 46.833333] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.838052] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 46.842764] EDAC amd64: MC: 4: 0MB 5: 0MB [ 46.847480] EDAC amd64: MC: 6: 0MB 7: 0MB [ 46.852193] EDAC amd64: using x8 syndromes. [ 46.856388] EDAC amd64: MCT channel count: 2 [ 46.880182] EDAC MC1: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:19.3 [ 46.887599] EDAC amd64: DRAM ECC enabled. [ 46.891623] EDAC amd64: F17h detected (node 2). [ 46.896206] EDAC MC: UMC0 chip selects: [ 46.896208] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.900916] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 46.905630] EDAC amd64: MC: 4: 0MB 5: 0MB [ 46.910336] EDAC amd64: MC: 6: 0MB 7: 0MB [ 46.915044] EDAC MC: UMC1 chip selects: [ 46.915045] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.919755] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 46.924462] EDAC amd64: MC: 4: 0MB 5: 0MB [ 46.929168] EDAC amd64: MC: 6: 0MB 7: 0MB [ 46.933875] EDAC amd64: using x8 syndromes. [ 46.938067] EDAC amd64: MCT channel count: 2 [ 46.955823] EDAC MC2: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1a.3 [ 46.963247] EDAC amd64: DRAM ECC enabled. [ 46.967274] EDAC amd64: F17h detected (node 3). [ 46.971864] EDAC MC: UMC0 chip selects: [ 46.971868] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.976579] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 46.981289] EDAC amd64: MC: 4: 0MB 5: 0MB [ 46.985998] EDAC amd64: MC: 6: 0MB 7: 0MB [ 46.990716] EDAC MC: UMC1 chip selects: [ 46.990718] EDAC amd64: MC: 0: 0MB 1: 0MB [ 46.995437] EDAC amd64: MC: 2: 16383MB 3: 16383MB [ 47.000151] EDAC amd64: MC: 4: 0MB 5: 0MB [ 47.004866] EDAC amd64: MC: 6: 0MB 7: 0MB [ 47.009579] EDAC amd64: using x8 syndromes. [ 47.013774] EDAC amd64: MCT channel count: 2 [ 47.026734] EDAC MC3: Giving out device to 'amd64_edac' 'F17h': DEV 0000:00:1b.3 [ 47.034555] EDAC PCI0: Giving out device to module 'amd64_edac' controller 'EDAC PCI controller': DEV '0000:00:18.0' (POLLED) [ 48.009432] dcdbas dcdbas: Dell Systems Management Base Driver (version 5.6.0-3.3) [ 73.354322] Adding 4194300k swap on /dev/sda3. Priority:-2 extents:1 across:4194300k FS [ 73.395961] type=1305 audit(1584563493.884:3): audit_pid=32789 old=0 auid=4294967295 ses=4294967295 res=1 [ 73.418996] RPC: Registered named UNIX socket transport module. [ 73.425535] RPC: Registered udp transport module. [ 73.431625] RPC: Registered tcp transport module. [ 73.437719] RPC: Registered tcp NFSv4.1 backchannel transport module. [ 74.078964] mlx5_core 0000:01:00.0: slow_pci_heuristic:5575:(pid 33065): Max link speed = 100000, PCI BW = 126016 [ 74.089289] mlx5_core 0000:01:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(256) RxCqeCmprss(0) [ 74.097563] mlx5_core 0000:01:00.0: MLX5E: StrdRq(0) RqSz(1024) StrdSz(256) RxCqeCmprss(0) [ 74.559915] tg3 0000:81:00.0: irq 254 for MSI/MSI-X [ 74.559934] tg3 0000:81:00.0: irq 255 for MSI/MSI-X [ 74.559946] tg3 0000:81:00.0: irq 256 for MSI/MSI-X [ 74.559958] tg3 0000:81:00.0: irq 257 for MSI/MSI-X [ 74.559968] tg3 0000:81:00.0: irq 258 for MSI/MSI-X [ 74.686056] IPv6: ADDRCONF(NETDEV_UP): em1: link is not ready [ 78.161541] tg3 0000:81:00.0 em1: Link is up at 1000 Mbps, full duplex [ 78.168073] tg3 0000:81:00.0 em1: Flow control is on for TX and on for RX [ 78.174864] tg3 0000:81:00.0 em1: EEE is enabled [ 78.179499] IPv6: ADDRCONF(NETDEV_CHANGE): em1: link becomes ready [ 79.040665] IPv6: ADDRCONF(NETDEV_UP): ib0: link is not ready [ 79.320147] IPv6: ADDRCONF(NETDEV_CHANGE): ib0: link becomes ready [ 83.455825] FS-Cache: Loaded [ 83.486699] FS-Cache: Netfs 'nfs' registered for caching [ 83.496188] Key type dns_resolver registered [ 83.524890] NFS: Registering the id_resolver key type [ 83.529951] Key type id_resolver registered [ 83.534144] Key type id_legacy registered [ 704.413883] LNet: HW NUMA nodes: 4, HW CPU cores: 48, npartitions: 4 [ 704.421368] alg: No test for adler32 (adler32-zlib) [ 705.222148] Lustre: Lustre: Build Version: 2.12.4 [ 705.326883] LNet: 47306:0:(config.c:1627:lnet_inet_enumerate()) lnet: Ignoring interface em2: it's down [ 705.336638] LNet: Using FastReg for registration [ 705.354010] LNet: Added LNI 10.0.10.109@o2ib7 [8/256/0/180] [ 967.224856] md: md8 stopped. [ 967.250869] async_tx: api initialized (async) [ 967.257055] xor: automatically using best checksumming function: [ 967.272536] avx : 9568.000 MB/sec [ 967.304520] raid6: sse2x1 gen() 6046 MB/s [ 967.325523] raid6: sse2x2 gen() 11378 MB/s [ 967.346518] raid6: sse2x4 gen() 12976 MB/s [ 967.367518] raid6: avx2x1 gen() 14250 MB/s [ 967.388560] raid6: avx2x2 gen() 18902 MB/s [ 967.409519] raid6: avx2x4 gen() 18902 MB/s [ 967.413796] raid6: using algorithm avx2x2 gen() (18902 MB/s) [ 967.419454] raid6: using avx2x2 recovery algorithm [ 967.440845] md/raid:md8: device dm-44 operational as raid disk 0 [ 967.446863] md/raid:md8: device dm-31 operational as raid disk 9 [ 967.452877] md/raid:md8: device dm-35 operational as raid disk 8 [ 967.458891] md/raid:md8: device dm-37 operational as raid disk 7 [ 967.464907] md/raid:md8: device dm-30 operational as raid disk 6 [ 967.470923] md/raid:md8: device dm-24 operational as raid disk 5 [ 967.476936] md/raid:md8: device dm-19 operational as raid disk 4 [ 967.482949] md/raid:md8: device dm-14 operational as raid disk 3 [ 967.488967] md/raid:md8: device dm-45 operational as raid disk 2 [ 967.494980] md/raid:md8: device dm-10 operational as raid disk 1 [ 967.502093] md/raid:md8: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 967.549539] md8: detected capacity change from 0 to 64011422924800 [ 967.557945] md: md0 stopped. [ 967.569047] md/raid:md0: device dm-118 operational as raid disk 0 [ 967.575387] md/raid:md0: device dm-94 operational as raid disk 9 [ 967.581402] md/raid:md0: device dm-89 operational as raid disk 8 [ 967.587415] md/raid:md0: device dm-66 operational as raid disk 7 [ 967.593422] md/raid:md0: device dm-85 operational as raid disk 6 [ 967.599436] md/raid:md0: device dm-72 operational as raid disk 5 [ 967.605442] md/raid:md0: device dm-51 operational as raid disk 4 [ 967.611447] md/raid:md0: device dm-93 operational as raid disk 3 [ 967.617453] md/raid:md0: device dm-83 operational as raid disk 2 [ 967.623460] md/raid:md0: device dm-104 operational as raid disk 1 [ 967.630259] md/raid:md0: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 967.677636] md0: detected capacity change from 0 to 64011422924800 [ 967.684838] md: md4 stopped. [ 967.694493] md/raid:md4: device dm-107 operational as raid disk 0 [ 967.700594] md/raid:md4: device dm-99 operational as raid disk 9 [ 967.706602] md/raid:md4: device dm-97 operational as raid disk 8 [ 967.712616] md/raid:md4: device dm-81 operational as raid disk 7 [ 967.718628] md/raid:md4: device dm-84 operational as raid disk 6 [ 967.724635] md/raid:md4: device dm-88 operational as raid disk 5 [ 967.730648] md/raid:md4: device dm-82 operational as raid disk 4 [ 967.736656] md/raid:md4: device dm-92 operational as raid disk 3 [ 967.742662] md/raid:md4: device dm-116 operational as raid disk 2 [ 967.748764] md/raid:md4: device dm-114 operational as raid disk 1 [ 967.755593] md/raid:md4: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 967.798694] md4: detected capacity change from 0 to 64011422924800 [ 967.807111] md: md10 stopped. [ 967.818180] md/raid:md10: device dm-11 operational as raid disk 0 [ 967.824323] md/raid:md10: device dm-63 operational as raid disk 9 [ 967.830444] md/raid:md10: device dm-36 operational as raid disk 8 [ 967.836582] md/raid:md10: device dm-56 operational as raid disk 7 [ 967.842703] md/raid:md10: device dm-39 operational as raid disk 6 [ 967.848861] md/raid:md10: device dm-28 operational as raid disk 5 [ 967.854985] md/raid:md10: device dm-40 operational as raid disk 4 [ 967.861104] md/raid:md10: device dm-4 operational as raid disk 3 [ 967.867124] md/raid:md10: device dm-3 operational as raid disk 2 [ 967.873180] md/raid:md10: device dm-12 operational as raid disk 1 [ 967.880144] md/raid:md10: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 967.919911] md10: detected capacity change from 0 to 64011422924800 [ 967.930282] md: md2 stopped. [ 967.943283] md/raid:md2: device dm-112 operational as raid disk 0 [ 967.949385] md/raid:md2: device dm-91 operational as raid disk 9 [ 967.955399] md/raid:md2: device dm-101 operational as raid disk 8 [ 967.961501] md/raid:md2: device dm-75 operational as raid disk 7 [ 967.967510] md/raid:md2: device dm-79 operational as raid disk 6 [ 967.973530] md/raid:md2: device dm-78 operational as raid disk 5 [ 967.979546] md/raid:md2: device dm-59 operational as raid disk 4 [ 967.985564] md/raid:md2: device dm-106 operational as raid disk 3 [ 967.991673] md/raid:md2: device dm-110 operational as raid disk 2 [ 967.997782] md/raid:md2: device dm-108 operational as raid disk 1 [ 968.005202] md/raid:md2: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 968.028375] md2: detected capacity change from 0 to 64011422924800 [ 968.039604] md: md6 stopped. [ 968.051408] md/raid:md6: device dm-0 operational as raid disk 0 [ 968.057347] md/raid:md6: device dm-25 operational as raid disk 9 [ 968.063372] md/raid:md6: device dm-58 operational as raid disk 8 [ 968.069401] md/raid:md6: device dm-53 operational as raid disk 7 [ 968.075421] md/raid:md6: device dm-21 operational as raid disk 6 [ 968.081444] md/raid:md6: device dm-8 operational as raid disk 5 [ 968.087381] md/raid:md6: device dm-7 operational as raid disk 4 [ 968.093313] md/raid:md6: device dm-43 operational as raid disk 3 [ 968.099334] md/raid:md6: device dm-55 operational as raid disk 2 [ 968.105347] md/raid:md6: device dm-54 operational as raid disk 1 [ 968.112559] md/raid:md6: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 968.161543] md6: detected capacity change from 0 to 64011422924800 [ 968.400753] LDISKFS-fs (md8): file extents enabled, maximum tree depth=5 [ 968.700765] LDISKFS-fs (md0): file extents enabled, maximum tree depth=5 [ 968.707735] LDISKFS-fs (md8): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 968.722788] LDISKFS-fs (md4): file extents enabled, maximum tree depth=5 [ 969.018562] LDISKFS-fs (md0): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 969.065883] LDISKFS-fs (md4): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 969.077790] LDISKFS-fs (md10): file extents enabled [ 969.077809] LDISKFS-fs (md2): file extents enabled [ 969.077811] , maximum tree depth=5 [ 969.089223] , maximum tree depth=5 [ 969.258025] LustreError: 137-5: fir-OST0030_UUID: not available for connect from 10.50.5.31@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 969.275313] LustreError: Skipped 11 previous similar messages [ 969.289785] LDISKFS-fs (md6): file extents enabled, maximum tree depth=5 [ 969.428449] LDISKFS-fs (md10): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 969.429290] LDISKFS-fs (md2): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 969.523560] Lustre: fir-OST0038: Not available for connect from 10.50.4.57@o2ib2 (not set up) [ 969.584132] LDISKFS-fs (md6): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 969.942260] LustreError: 137-5: fir-OST0030_UUID: not available for connect from 10.50.7.49@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 969.959547] LustreError: Skipped 49 previous similar messages [ 970.054266] Lustre: fir-OST0038: Not available for connect from 10.50.1.60@o2ib2 (not set up) [ 970.062810] Lustre: Skipped 5 previous similar messages [ 970.639055] Lustre: fir-OST0038: Imperative Recovery enabled, recovery window shrunk from 300-900 down to 150-900 [ 970.650027] Lustre: fir-OST0038: in recovery but waiting for the first client to connect [ 970.674225] Lustre: fir-OST0038: Will be in recovery for at least 2:30, or until 1291 clients reconnect [ 970.683666] Lustre: fir-OST0038: Connection restored to (at 10.49.24.8@o2ib1) [ 970.957503] LustreError: 137-5: fir-OST0030_UUID: not available for connect from 10.50.2.31@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 970.974815] LustreError: Skipped 55 previous similar messages [ 971.183149] Lustre: fir-OST0030: Imperative Recovery enabled, recovery window shrunk from 300-900 down to 150-900 [ 971.193412] Lustre: Skipped 1 previous similar message [ 971.199299] Lustre: fir-OST0030: in recovery but waiting for the first client to connect [ 971.207400] Lustre: Skipped 1 previous similar message [ 971.262473] Lustre: fir-OST0030: Will be in recovery for at least 2:30, or until 1291 clients reconnect [ 971.262784] Lustre: fir-OST0038: Connection restored to (at 10.50.2.58@o2ib2) [ 971.262785] Lustre: fir-OST0034: Connection restored to (at 10.50.2.58@o2ib2) [ 971.262786] Lustre: Skipped 11 previous similar messages [ 971.262787] Lustre: Skipped 11 previous similar messages [ 971.296978] Lustre: Skipped 1 previous similar message [ 971.370814] Lustre: fir-OST003a: Not available for connect from 10.50.4.46@o2ib2 (not set up) [ 971.379344] Lustre: Skipped 6 previous similar messages [ 973.523001] Lustre: fir-OST003a: Connection restored to 5bccfc24-b020-4 (at 10.50.3.47@o2ib2) [ 973.531533] Lustre: Skipped 11 previous similar messages [ 978.338635] Lustre: fir-OST0038: Connection restored to b7c30a0f-2fff-4 (at 10.49.8.23@o2ib1) [ 987.314228] Lustre: fir-OST0034: Connection restored to e8c40140-31a1-4 (at 10.49.28.9@o2ib1) [ 987.322775] Lustre: Skipped 8 previous similar messages [ 989.040856] Lustre: fir-OST0036: Will be in recovery for at least 2:30, or until 1291 clients reconnect [ 989.050265] Lustre: Skipped 2 previous similar messages [ 1003.321453] Lustre: fir-OST0030: Connection restored to 6058db13-014f-4 (at 10.50.5.19@o2ib2) [ 1003.329985] Lustre: Skipped 1817 previous similar messages [ 1038.463041] Lustre: fir-OST0034: deleting orphan objects from 0x1000000400:21380459 to 0x1000000400:21380481 [ 1038.470141] Lustre: fir-OST0034: Recovery over after 1:07, of 1291 clients 1291 recovered and 0 were evicted. [ 1038.470432] Lustre: fir-OST003a: deleting orphan objects from 0x1180000400:21370876 to 0x1180000400:21370897 [ 1038.482473] Lustre: fir-OST0034: deleting orphan objects from 0x1000000402:14878387 to 0x1000000402:14878433 [ 1038.506598] Lustre: fir-OST0030: deleting orphan objects from 0xf00000402:14928741 to 0xf00000402:14928769 [ 1038.523362] Lustre: fir-OST0030: deleting orphan objects from 0xf00000400:21510628 to 0xf00000400:21510657 [ 1038.546418] Lustre: fir-OST0036: deleting orphan objects from 0x1080000400:21260263 to 0x1080000400:21260289 [ 1038.548039] Lustre: fir-OST0032: deleting orphan objects from 0xf80000401:21172903 to 0xf80000401:21172929 [ 1038.564676] Lustre: fir-OST003a: deleting orphan objects from 0x1180000402:15005239 to 0x1180000402:15005281 [ 1038.571278] Lustre: fir-OST0030: deleting orphan objects from 0xf00000401:7872226 to 0xf00000401:7872289 [ 1038.576208] Lustre: fir-OST0034: deleting orphan objects from 0x1000000401:7802194 to 0x1000000401:7802209 [ 1038.591627] Lustre: fir-OST0034: deleting orphan objects from 0x0:92671500 to 0x0:92671521 [ 1038.609615] Lustre: fir-OST0038: deleting orphan objects from 0x1100000401:14949614 to 0x1100000401:14949633 [ 1038.613179] Lustre: fir-OST003a: deleting orphan objects from 0x1180000401:7893311 to 0x1180000401:7893377 [ 1038.613242] Lustre: fir-OST003a: deleting orphan objects from 0x0:93215500 to 0x0:93215521 [ 1038.631094] Lustre: fir-OST0036: deleting orphan objects from 0x1080000402:14939055 to 0x1080000402:14939073 [ 1038.633473] Lustre: fir-OST0038: deleting orphan objects from 0x1100000402:7865419 to 0x1100000402:7865505 [ 1038.634235] Lustre: fir-OST0038: deleting orphan objects from 0x1100000400:21244891 to 0x1100000400:21244929 [ 1038.636503] Lustre: fir-OST0038: deleting orphan objects from 0x0:93306861 to 0x0:93306881 [ 1038.647776] Lustre: fir-OST0030: deleting orphan objects from 0x0:91198735 to 0x0:91198753 [ 1038.672168] Lustre: fir-OST0036: deleting orphan objects from 0x1080000401:7815657 to 0x1080000401:7815681 [ 1038.693539] Lustre: fir-OST0036: deleting orphan objects from 0x0:91542067 to 0x0:91542081 [ 1038.694237] Lustre: fir-OST0032: deleting orphan objects from 0xf80000402:7739628 to 0xf80000402:7739649 [ 1038.694312] Lustre: fir-OST0032: deleting orphan objects from 0xf80000400:14817963 to 0xf80000400:14817985 [ 1038.694399] Lustre: fir-OST0032: deleting orphan objects from 0x0:90614861 to 0x0:90614881 [ 1118.808396] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.50.6.25@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 1118.825699] LustreError: Skipped 12 previous similar messages [ 1122.794275] Lustre: fir-OST0032: Client a9fb14eb-f20e-4 (at 10.50.17.6@o2ib2) reconnecting [ 1122.802567] Lustre: fir-OST0032: Connection restored to a9fb14eb-f20e-4 (at 10.50.17.6@o2ib2) [ 1122.811092] Lustre: Skipped 5459 previous similar messages [ 1122.853977] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.50.13.3@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 1122.871257] LustreError: Skipped 62 previous similar messages [ 1124.379514] Lustre: fir-OST0038: Client 65d3244f-eb29-4 (at 10.49.25.12@o2ib1) reconnecting [ 1126.735396] Lustre: fir-OST0036: Client eafb47de-37b1-4 (at 10.50.5.13@o2ib2) reconnecting [ 1129.789669] Lustre: fir-OST0034: Client 8c909327-41ba-4 (at 10.50.6.17@o2ib2) reconnecting [ 1130.964920] LustreError: 137-5: fir-OST0033_UUID: not available for connect from 10.50.8.29@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 1130.982233] LustreError: Skipped 101 previous similar messages [ 1134.035454] Lustre: fir-OST0030: Client 7baf8117-42a6-4 (at 10.50.9.9@o2ib2) reconnecting [ 1134.043634] Lustre: Skipped 1 previous similar message [ 1137.582057] md: md1 stopped. [ 1137.591984] md/raid:md1: device dm-52 operational as raid disk 0 [ 1137.597997] md/raid:md1: device dm-90 operational as raid disk 9 [ 1137.604012] md/raid:md1: device dm-100 operational as raid disk 8 [ 1137.610134] md/raid:md1: device dm-74 operational as raid disk 7 [ 1137.616150] md/raid:md1: device dm-68 operational as raid disk 6 [ 1137.622165] md/raid:md1: device dm-60 operational as raid disk 5 [ 1137.628176] md/raid:md1: device dm-70 operational as raid disk 4 [ 1137.634192] md/raid:md1: device dm-96 operational as raid disk 3 [ 1137.640220] md/raid:md1: device dm-98 operational as raid disk 2 [ 1137.646227] md/raid:md1: device dm-109 operational as raid disk 1 [ 1137.652874] md/raid:md1: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 1137.680907] md1: detected capacity change from 0 to 64011422924800 [ 1137.688198] md: md7 stopped. [ 1137.698361] md/raid:md7: device dm-1 operational as raid disk 0 [ 1137.704289] md/raid:md7: device dm-46 operational as raid disk 9 [ 1137.710301] md/raid:md7: device dm-33 operational as raid disk 8 [ 1137.716314] md/raid:md7: device dm-34 operational as raid disk 7 [ 1137.722328] md/raid:md7: device dm-29 operational as raid disk 6 [ 1137.728334] md/raid:md7: device dm-42 operational as raid disk 5 [ 1137.734339] md/raid:md7: device dm-9 operational as raid disk 4 [ 1137.740261] md/raid:md7: device dm-32 operational as raid disk 3 [ 1137.746275] md/raid:md7: device dm-61 operational as raid disk 2 [ 1137.752289] md/raid:md7: device dm-71 operational as raid disk 1 [ 1137.758899] md/raid:md7: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 1137.779473] md7: detected capacity change from 0 to 64011422924800 [ 1137.786998] md: md11 stopped. [ 1137.800782] md/raid:md11: device dm-13 operational as raid disk 0 [ 1137.806879] md/raid:md11: device dm-38 operational as raid disk 9 [ 1137.813004] md/raid:md11: device dm-64 operational as raid disk 8 [ 1137.819136] md/raid:md11: device dm-16 operational as raid disk 7 [ 1137.825246] md/raid:md11: device dm-57 operational as raid disk 6 [ 1137.831359] md/raid:md11: device dm-15 operational as raid disk 5 [ 1137.837507] md/raid:md11: device dm-48 operational as raid disk 4 [ 1137.843612] md/raid:md11: device dm-6 operational as raid disk 3 [ 1137.849625] md/raid:md11: device dm-5 operational as raid disk 2 [ 1137.855641] md/raid:md11: device dm-27 operational as raid disk 1 [ 1137.862411] md/raid:md11: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 1137.883453] md11: detected capacity change from 0 to 64011422924800 [ 1137.892722] md: md5 stopped. [ 1137.906524] md/raid:md5: device dm-113 operational as raid disk 0 [ 1137.912627] md/raid:md5: device dm-105 operational as raid disk 9 [ 1137.918732] md/raid:md5: device dm-102 operational as raid disk 8 [ 1137.924838] md/raid:md5: device dm-86 operational as raid disk 7 [ 1137.930857] md/raid:md5: device dm-87 operational as raid disk 6 [ 1137.936871] md/raid:md5: device dm-80 operational as raid disk 5 [ 1137.942888] md/raid:md5: device dm-76 operational as raid disk 4 [ 1137.948901] md/raid:md5: device dm-119 operational as raid disk 3 [ 1137.955016] md/raid:md5: device dm-67 operational as raid disk 2 [ 1137.961047] md/raid:md5: device dm-69 operational as raid disk 1 [ 1137.967717] md/raid:md5: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 1137.998219] md5: detected capacity change from 0 to 64011422924800 [ 1138.016379] md: md9 stopped. [ 1138.033154] md/raid:md9: device dm-18 operational as raid disk 0 [ 1138.039204] md/raid:md9: device dm-23 operational as raid disk 9 [ 1138.045234] md/raid:md9: device dm-26 operational as raid disk 8 [ 1138.051260] md/raid:md9: device dm-49 operational as raid disk 7 [ 1138.057289] md/raid:md9: device dm-41 operational as raid disk 6 [ 1138.063314] md/raid:md9: device dm-50 operational as raid disk 5 [ 1138.069337] md/raid:md9: device dm-22 operational as raid disk 4 [ 1138.075358] md/raid:md9: device dm-2 operational as raid disk 3 [ 1138.081292] md/raid:md9: device dm-17 operational as raid disk 2 [ 1138.087313] md/raid:md9: device dm-20 operational as raid disk 1 [ 1138.094200] md/raid:md9: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 1138.114750] md9: detected capacity change from 0 to 64011422924800 [ 1138.132097] md: md3 stopped. [ 1138.143640] md/raid:md3: device dm-117 operational as raid disk 0 [ 1138.149766] md/raid:md3: device dm-95 operational as raid disk 9 [ 1138.155778] md/raid:md3: device dm-103 operational as raid disk 8 [ 1138.161876] md/raid:md3: device dm-73 operational as raid disk 7 [ 1138.167894] md/raid:md3: device dm-77 operational as raid disk 6 [ 1138.173913] md/raid:md3: device dm-65 operational as raid disk 5 [ 1138.179930] md/raid:md3: device dm-62 operational as raid disk 4 [ 1138.185945] md/raid:md3: device dm-47 operational as raid disk 3 [ 1138.191964] md/raid:md3: device dm-111 operational as raid disk 2 [ 1138.198062] md/raid:md3: device dm-115 operational as raid disk 1 [ 1138.204980] md/raid:md3: raid level 6 active with 10 out of 10 devices, algorithm 2 [ 1138.256511] md3: detected capacity change from 0 to 64011422924800 [ 1138.439398] LDISKFS-fs (md1): file extents enabled, maximum tree depth=5 [ 1138.702394] LDISKFS-fs (md7): file extents enabled, maximum tree depth=5 [ 1138.768062] LDISKFS-fs (md1): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 1138.892393] LDISKFS-fs (md11): file extents enabled, maximum tree depth=5 [ 1139.013010] LDISKFS-fs (md7): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 1139.239844] LDISKFS-fs (md11): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 1139.365382] LDISKFS-fs (md3): file extents enabled [ 1139.367440] LDISKFS-fs (md9): file extents enabled [ 1139.367442] , maximum tree depth=5 [ 1139.376728] , maximum tree depth=5 [ 1139.397422] LDISKFS-fs (md5): file extents enabled, maximum tree depth=5 [ 1139.545454] Lustre: fir-OST0031: Imperative Recovery enabled, recovery window shrunk from 300-900 down to 150-900 [ 1139.555724] Lustre: Skipped 3 previous similar messages [ 1139.564512] Lustre: fir-OST0031: in recovery but waiting for the first client to connect [ 1139.572606] Lustre: Skipped 3 previous similar messages [ 1139.613413] Lustre: fir-OST0031: Will be in recovery for at least 2:30, or until 1291 clients reconnect [ 1139.693934] LDISKFS-fs (md9): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 1139.721872] LDISKFS-fs (md3): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 1139.733116] LDISKFS-fs (md5): mounted filesystem with ordered data mode. Opts: errors=remount-ro,no_mbcache,nodelalloc [ 1140.144518] Lustre: fir-OST003b: Not available for connect from 10.49.30.25@o2ib1 (not set up) [ 1140.153140] Lustre: Skipped 1 previous similar message [ 1145.842176] Lustre: fir-OST0038: Client 18894ca4-7e93-4 (at 10.50.3.20@o2ib2) reconnecting [ 1145.850447] Lustre: Skipped 3 previous similar messages [ 1151.609971] Lustre: fir-OST0037: Client c89129d8-949d-4 (at 10.49.30.18@o2ib1) reconnected, waiting for 1291 clients in recovery for 2:30 [ 1153.437200] Lustre: fir-OST003b: Client a2dc795b-2f2c-4 (at 10.49.26.5@o2ib1) reconnected, waiting for 1291 clients in recovery for 2:28 [ 1153.449460] Lustre: Skipped 2 previous similar messages [ 1161.713033] Lustre: fir-OST0037: Client 72a94371-b86b-4 (at 10.50.9.67@o2ib2) reconnected, waiting for 1291 clients in recovery for 2:20 [ 1161.725285] Lustre: Skipped 1 previous similar message [ 1164.959650] Lustre: fir-OST0034: Client 376f46a0-8125-4 (at 10.50.5.18@o2ib2) reconnecting [ 1164.967919] Lustre: Skipped 21 previous similar messages [ 1180.364080] LNetError: 47371:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.0.10.224@o2ib7 added to recovery queue. Health = 900 [ 1180.364093] LustreError: 49250:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 3145728(4194304) req@ffff9feb9d56d050 x1659478365626176/t0(0) o4->3e2b2629-3d90-4@10.50.4.52@o2ib2:64/0 lens 488/448 e 1 to 0 dl 1584564619 ref 1 fl Interpret:/0/0 rc 0/0 [ 1180.364200] Lustre: fir-OST0036: Bulk IO write error with 3e2b2629-3d90-4 (at 10.50.4.52@o2ib2), client will retry: rc = -110 [ 1180.411968] LustreError: 49386:0:(ldlm_lib.c:3294:target_bulk_io()) @@@ truncated bulk READ 6291456(8388608) req@ffff9ffb77caa050 x1659468117680256/t0(0) o3->f5c72d15-474b-4@10.50.1.8@o2ib2:52/0 lens 488/440 e 1 to 0 dl 1584564607 ref 1 fl Interpret:/0/0 rc 0/0 [ 1180.435318] Lustre: fir-OST0034: Bulk IO read error with f5c72d15-474b-4 (at 10.50.1.8@o2ib2), client will retry: rc -110 [ 1181.562325] Lustre: fir-OST0031: Client 3ab8a3cd-4496-4 (at 10.49.19.3@o2ib1) reconnected, waiting for 1291 clients in recovery for 1:59 [ 1186.002985] Lustre: fir-OST0039: Client 258a96a8-92df-4 (at 10.49.24.18@o2ib1) reconnected, waiting for 1291 clients in recovery for 1:55 [ 1186.015328] Lustre: Skipped 3 previous similar messages [ 1187.176305] Lustre: fir-OST0037: Connection restored to 7a5838a5-d04c-4 (at 10.50.7.25@o2ib2) [ 1187.184846] Lustre: Skipped 1282 previous similar messages [ 1195.843699] Lustre: fir-OST0037: Client 765b7c94-79a5-4 (at 10.50.6.53@o2ib2) reconnected, waiting for 1291 clients in recovery for 1:45 [ 1195.855953] Lustre: Skipped 2 previous similar messages [ 1197.177082] Lustre: fir-OST003a: Client b72db314-9e65-4 (at 10.50.5.4@o2ib2) reconnecting [ 1197.185261] Lustre: Skipped 55 previous similar messages [ 1205.412281] LNetError: 47371:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.0.10.226@o2ib7 added to recovery queue. Health = 900 [ 1205.425245] LNetError: 47371:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 1 previous similar message [ 1205.435969] LustreError: 49256:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(3334144) req@ffff9feb78268050 x1660631857211136/t0(0) o4->a5507c8e-5adc-4@10.50.10.71@o2ib2:90/0 lens 488/448 e 1 to 0 dl 1584564645 ref 1 fl Interpret:/0/0 rc 0/0 [ 1205.459062] LustreError: 49256:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 1 previous similar message [ 1205.468842] Lustre: fir-OST0036: Bulk IO write error with a5507c8e-5adc-4 (at 10.50.10.71@o2ib2), client will retry: rc = -110 [ 1205.480252] Lustre: Skipped 1 previous similar message [ 1212.588480] Lustre: fir-OST0037: Client 3262f122-732c-4 (at 10.49.20.13@o2ib1) reconnected, waiting for 1291 clients in recovery for 1:29 [ 1212.600825] Lustre: Skipped 6 previous similar messages [ 1229.435980] LustreError: 49148:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(17089) req@ffffa00b9274d050 x1660618376631360/t0(0) o4->1b03a84c-13fb-4@10.50.8.44@o2ib2:114/0 lens 488/448 e 2 to 0 dl 1584564669 ref 1 fl Interpret:/0/0 rc 0/0 [ 1229.458937] Lustre: fir-OST0036: Bulk IO write error with 1b03a84c-13fb-4 (at 10.50.8.44@o2ib2), client will retry: rc = -110 [ 1246.561193] Lustre: fir-OST0031: Client 2a27cfd8-18f1-4 (at 10.49.23.25@o2ib1) reconnected, waiting for 1291 clients in recovery for 0:54 [ 1246.573555] Lustre: Skipped 37 previous similar messages [ 1261.561342] Lustre: fir-OST0030: Client a489af8f-6baa-4 (at 10.49.23.28@o2ib1) reconnecting [ 1261.569696] Lustre: Skipped 193 previous similar messages [ 1279.435854] LNetError: 47371:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.0.10.225@o2ib7 added to recovery queue. Health = 900 [ 1279.448803] LNetError: 47371:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 2 previous similar messages [ 1279.459638] LustreError: 48328:0:(ldlm_lib.c:3294:target_bulk_io()) @@@ truncated bulk READ 0(8388608) req@ffff9fdb32762850 x1659150350679040/t0(0) o3->7a4b5ab1-0a05-4@10.50.10.40@o2ib2:150/0 lens 488/440 e 1 to 0 dl 1584564705 ref 1 fl Interpret:/0/0 rc 0/0 [ 1279.459661] LustreError: 49023:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) @@@ truncated bulk GET 0(2162688) req@ffff9ffb76da9850 x1660631857211072/t0(0) o4->a5507c8e-5adc-4@10.50.10.71@o2ib2:172/0 lens 488/448 e 2 to 0 dl 1584564727 ref 1 fl Interpret:/0/0 rc 0/0 [ 1279.459664] LustreError: 49023:0:(sec.c:2485:sptlrpc_svc_unwrap_bulk()) Skipped 1 previous similar message [ 1279.459743] Lustre: fir-OST0038: Bulk IO write error with a5507c8e-5adc-4 (at 10.50.10.71@o2ib2), client will retry: rc = -110 [ 1279.459884] Lustre: fir-OST0034: Bulk IO read error with cbf4c94a-83d4-4 (at 10.50.5.61@o2ib2), client will retry: rc -110 [ 1279.459886] Lustre: Skipped 1 previous similar message [ 1279.542974] LustreError: 48328:0:(ldlm_lib.c:3294:target_bulk_io()) Skipped 2 previous similar messages [ 1279.887661] Lustre: fir-OST0033: Client 6535b1ff-9b91-4 (at 10.49.21.36@o2ib1) refused connection, still busy with 6 references [ 1280.600424] Lustre: fir-OST003b: Client 556dc377-06e7-4 (at 10.49.24.33@o2ib1) refused connection, still busy with 6 references [ 1280.610855] LustreError: 49074:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff9fdb32752050 x1659263884797440/t0(0) o4->3ebd75af-84f9-4@10.50.7.36@o2ib2:190/0 lens 488/448 e 0 to 0 dl 1584564745 ref 1 fl Interpret:/0/0 rc 0/0 [ 1280.634121] Lustre: Skipped 4 previous similar messages [ 1281.547848] LustreError: 49148:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk READ req@ffffa00b93528850 x1660618376633664/t0(0) o3->1b03a84c-13fb-4@10.50.8.44@o2ib2:191/0 lens 488/440 e 0 to 0 dl 1584564746 ref 1 fl Interpret:/0/0 rc 0/0 [ 1281.570007] LustreError: 49148:0:(ldlm_lib.c:3279:target_bulk_io()) Skipped 1 previous similar message [ 1281.579486] Lustre: fir-OST003a: Bulk IO read error with 1b03a84c-13fb-4 (at 10.50.8.44@o2ib2), client will retry: rc -110 [ 1281.590562] Lustre: Skipped 1 previous similar message [ 1281.599991] Lustre: fir-OST0035: Client 46f8aff0-e02f-4 (at 10.49.26.29@o2ib1) refused connection, still busy with 6 references [ 1281.611460] Lustre: Skipped 25 previous similar messages [ 1283.600486] Lustre: fir-OST0033: Client c2f3b7ea-3e93-4 (at 10.49.13.28@o2ib1) refused connection, still busy with 6 references [ 1283.611958] Lustre: Skipped 18 previous similar messages [ 1285.200842] LustreError: 49325:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk READ req@ffff9ffb7879e050 x1659191407329344/t0(0) o3->f9de01a6-d260-4@10.50.8.54@o2ib2:174/0 lens 488/440 e 0 to 0 dl 1584564729 ref 1 fl Interpret:/0/0 rc 0/0 [ 1285.215998] Lustre: fir-OST0032: Bulk IO read error with f9de01a6-d260-4 (at 10.50.8.54@o2ib2), client will retry: rc -110 [ 1285.234009] LustreError: 49325:0:(ldlm_lib.c:3279:target_bulk_io()) Skipped 2 previous similar messages [ 1297.569779] Lustre: fir-OST003b: Client 13e7c6de-0e36-4 (at 10.49.27.30@o2ib1) refused connection, still busy with 6 references [ 1297.581278] Lustre: Skipped 114 previous similar messages [ 1299.795886] Lustre: fir-OST0031: Recovery already passed deadline 0:00. If you do not want to wait more, you may force taget eviction via 'lctl --device fir-OST0031 abort_recovery. [ 1299.811960] Lustre: Skipped 1 previous similar message [ 1300.341073] Lustre: fir-OST0031: Recovery already passed deadline 0:00. If you do not want to wait more, you may force taget eviction via 'lctl --device fir-OST0031 abort_recovery. [ 1300.357162] Lustre: Skipped 12 previous similar messages [ 1300.763799] Lustre: fir-OST0031: recovery is timed out, evict stale exports [ 1300.771022] Lustre: fir-OST0031: disconnecting 145 stale clients [ 1300.896862] Lustre: fir-OST0031: Denying connection for new client 65d3244f-eb29-4 (at 10.49.25.12@o2ib1), waiting for 1291 known clients (671 recovered, 475 in progress, and 145 evicted) to recover in 0:59 [ 1301.360255] Lustre: fir-OST0037: Recovery already passed deadline 0:00. If you do not want to wait more, you may force taget eviction via 'lctl --device fir-OST0037 abort_recovery. [ 1301.376320] Lustre: Skipped 60 previous similar messages [ 1301.586325] Lustre: fir-OST0031: Denying connection for new client fc6fb97c-9e38-4 (at 10.49.25.29@o2ib1), waiting for 1291 known clients (684 recovered, 462 in progress, and 145 evicted) to recover in 0:59 [ 1301.604662] Lustre: Skipped 3 previous similar messages [ 1301.763794] Lustre: fir-OST003b: recovery is timed out, evict stale exports [ 1301.763985] Lustre: fir-OST0037: disconnecting 112 stale clients [ 1301.776771] Lustre: Skipped 1 previous similar message [ 1302.711769] Lustre: fir-OST0037: Denying connection for new client facaa254-332c-4 (at 10.50.17.11@o2ib2), waiting for 1291 known clients (710 recovered, 469 in progress, and 112 evicted) to recover in 0:59 [ 1302.730116] Lustre: Skipped 25 previous similar messages [ 1302.763789] Lustre: fir-OST0033: recovery is timed out, evict stale exports [ 1302.764028] Lustre: fir-OST0035: disconnecting 116 stale clients [ 1302.764030] Lustre: Skipped 2 previous similar messages [ 1302.782009] Lustre: Skipped 1 previous similar message [ 1304.733948] Lustre: fir-OST0035: Denying connection for new client 3d9f6753-d00b-4 (at 10.50.7.30@o2ib2), waiting for 1291 known clients (751 recovered, 424 in progress, and 116 evicted) to recover in 0:58 [ 1304.752180] Lustre: Skipped 84 previous similar messages [ 1308.825719] Lustre: fir-OST0033: Denying connection for new client 6734b517-1c96-4 (at 10.50.17.14@o2ib2), waiting for 1291 known clients (862 recovered, 306 in progress, and 123 evicted) to recover in 0:53 [ 1308.844042] Lustre: Skipped 146 previous similar messages [ 1310.581821] Lustre: fir-OST0033: Client efcc39a6-2179-4 (at 10.49.23.17@o2ib1) reconnected, waiting for 1291 clients in recovery for 0:52 [ 1310.594166] Lustre: Skipped 6474 previous similar messages [ 1313.430265] Lustre: fir-OST003a: haven't heard from client f9e02322-74d7-4 (at 10.50.8.16@o2ib2) in 224 seconds. I think it's dead, and I am evicting it. exp ffff9feb808e3800, cur 1584564734 expire 1584564584 last 1584564510 [ 1314.480460] Lustre: fir-OST0034: haven't heard from client fb2c1382-8f5a-4 (at 10.50.15.10@o2ib2) in 222 seconds. I think it's dead, and I am evicting it. exp ffff9febf59eac00, cur 1584564735 expire 1584564585 last 1584564513 [ 1314.500453] Lustre: Skipped 57 previous similar messages [ 1315.206060] Lustre: fir-OST0030: Connection restored to f3674d1a-1c6f-4 (at 10.50.5.55@o2ib2) [ 1315.214589] Lustre: Skipped 22643 previous similar messages [ 1316.450228] Lustre: fir-OST0036: haven't heard from client ecf003cb-b7be-4 (at 10.50.16.8@o2ib2) in 208 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d0fcc00, cur 1584564737 expire 1584564587 last 1584564529 [ 1316.470108] Lustre: Skipped 191 previous similar messages [ 1317.018270] Lustre: fir-OST0033: Denying connection for new client 2f0aac2c-926c-4 (at 10.50.2.62@o2ib2), waiting for 1291 known clients (985 recovered, 183 in progress, and 123 evicted) to recover in 0:45 [ 1317.036502] Lustre: Skipped 185 previous similar messages [ 1318.495954] Lustre: fir-OST0030: haven't heard from client 0c02c6b3-fbbf-4 (at 10.50.13.14@o2ib2) in 163 seconds. I think it's dead, and I am evicting it. exp ffff9feb7f517000, cur 1584564739 expire 1584564589 last 1584564576 [ 1318.515954] Lustre: Skipped 113 previous similar messages [ 1333.033282] Lustre: fir-OST0039: Denying connection for new client 2a442b89-d680-4 (at 10.50.6.7@o2ib2), waiting for 1291 known clients (1120 recovered, 67 in progress, and 104 evicted) to recover in 0:28 [ 1333.051471] Lustre: Skipped 433 previous similar messages [ 1360.763663] Lustre: fir-OST0031: recovery is timed out, evict stale exports [ 1360.770858] Lustre: fir-OST0031: disconnecting 5 stale clients [ 1360.776693] Lustre: Skipped 1 previous similar message [ 1360.975663] Lustre: fir-OST0031: Recovery over after 3:42, of 1291 clients 1141 recovered and 150 were evicted. [ 1360.985772] Lustre: Skipped 5 previous similar messages [ 1361.043451] Lustre: fir-OST0031: deleting orphan objects from 0xf40000402:14993242 to 0xf40000402:14993281 [ 1361.044909] Lustre: fir-OST0031: deleting orphan objects from 0xf40000401:21960993 to 0xf40000401:21961025 [ 1361.083308] Lustre: fir-OST0031: deleting orphan objects from 0x0:92752608 to 0x0:92752641 [ 1361.102577] Lustre: fir-OST0031: deleting orphan objects from 0xf40000400:7950954 to 0xf40000400:7951009 [ 1361.928542] Lustre: fir-OST0037: deleting orphan objects from 0x10c0000402:15012232 to 0x10c0000402:15012257 [ 1361.958754] Lustre: fir-OST0037: deleting orphan objects from 0x10c0000401:21483694 to 0x10c0000401:21483713 [ 1361.992728] Lustre: fir-OST0037: deleting orphan objects from 0x0:93572837 to 0x0:93572865 [ 1362.014053] Lustre: fir-OST0037: deleting orphan objects from 0x10c0000400:7916820 to 0x10c0000400:7916833 [ 1362.016214] Lustre: fir-OST0039: deleting orphan objects from 0x1140000401:21532469 to 0x1140000401:21532513 [ 1362.040794] Lustre: fir-OST0039: deleting orphan objects from 0x1140000400:15035067 to 0x1140000400:15035105 [ 1362.043951] Lustre: fir-OST0033: Recovery already passed deadline 0:00. If you do not want to wait more, you may force taget eviction via 'lctl --device fir-OST0033 abort_recovery. [ 1362.043952] Lustre: Skipped 55 previous similar messages [ 1362.050661] Lustre: fir-OST0039: deleting orphan objects from 0x1140000402:7943213 to 0x1140000402:7943233 [ 1362.058375] Lustre: fir-OST003b: deleting orphan objects from 0x11c0000400:21674192 to 0x11c0000400:21674209 [ 1362.060282] Lustre: fir-OST003b: deleting orphan objects from 0x11c0000401:15054991 to 0x11c0000401:15055009 [ 1362.101690] Lustre: fir-OST0039: deleting orphan objects from 0x0:94444864 to 0x0:94444897 [ 1362.152728] Lustre: fir-OST003b: deleting orphan objects from 0x11c0000402:7953790 to 0x11c0000402:7953825 [ 1362.177764] Lustre: fir-OST003b: deleting orphan objects from 0x0:93292731 to 0x0:93292769 [ 1363.433718] Lustre: fir-OST0035: Recovery over after 3:42, of 1291 clients 1168 recovered and 123 were evicted. [ 1363.443952] Lustre: Skipped 4 previous similar messages [ 1363.491804] Lustre: fir-OST0035: deleting orphan objects from 0x1040000400:21509123 to 0x1040000400:21509153 [ 1363.522094] Lustre: fir-OST0035: deleting orphan objects from 0x1040000401:7953757 to 0x1040000401:7953793 [ 1363.523776] Lustre: fir-OST0033: deleting orphan objects from 0xfc0000400:21670825 to 0xfc0000400:21670849 [ 1363.535720] Lustre: fir-OST0035: deleting orphan objects from 0x1040000402:14989705 to 0x1040000402:14989729 [ 1363.552792] Lustre: fir-OST0033: deleting orphan objects from 0xfc0000401:14941744 to 0xfc0000401:14941761 [ 1363.593482] Lustre: fir-OST0035: deleting orphan objects from 0x0:94185277 to 0x0:94185313 [ 1363.618871] Lustre: fir-OST0033: deleting orphan objects from 0xfc0000402:7946022 to 0xfc0000402:7946049 [ 1363.693028] Lustre: fir-OST0033: deleting orphan objects from 0x0:93959396 to 0x0:93959425 [ 1391.923808] Lustre: fir-OST0036: Client 22cf25db-5638-4 (at 10.49.30.7@o2ib1) reconnecting [ 1391.932079] Lustre: Skipped 11661 previous similar messages [ 1395.485441] LustreError: 49234:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0037: cli 3e2b2629-3d90-4 claims 16801792 GRANT, real grant 0 [ 1653.319860] LustreError: 48900:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0033: cli f7e23083-48a0-4 claims 126976 GRANT, real grant 73728 [ 1748.803524] LustreError: 49009:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0035: cli 8b0900a7-167c-4 claims 786432 GRANT, real grant 0 [ 1777.793609] LustreError: 49257:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003b: cli 20917503-c5d6-4 claims 770048 GRANT, real grant 0 [ 1793.051202] LustreError: 49243:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0037: cli 0dc74af1-84ad-4 claims 36864 GRANT, real grant 0 [ 1825.434597] LustreError: 49357:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0039: cli 7fefb458-5058-4 claims 28672 GRANT, real grant 0 [ 1871.012381] LustreError: 49198:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003b: cli 4171bc70-1760-4 claims 598016 GRANT, real grant 0 [ 1903.573819] LustreError: 49306:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0031: cli fec02622-e2c9-4 claims 32768 GRANT, real grant 0 [ 2079.694508] LustreError: 49074:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0031: cli b969ea1c-f652-4 claims 524288 GRANT, real grant 0 [ 2229.275071] LustreError: 49220:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0037: cli b3f0b304-1697-4 claims 1544192 GRANT, real grant 1495040 [ 2588.192196] LustreError: 49234:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 00c7f158-cc8b-4 claims 16801792 GRANT, real grant 16777216 [ 2588.205153] LustreError: 49234:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [ 3300.341500] Lustre: Failing over fir-OST0033 [ 3300.345843] Lustre: Skipped 2 previous similar messages [ 3300.380250] Lustre: fir-OST0031: Not available for connect from 10.50.4.68@o2ib2 (stopping) [ 3300.388640] Lustre: Skipped 7 previous similar messages [ 3300.441754] LustreError: 67756:0:(ldlm_resource.c:1147:ldlm_resource_complain()) filter-fir-OST0031_UUID: namespace resource [0xf40000402:0xe19ed6:0x0].0x0 (ffffa00a286849c0) refcount nonzero (2) after lock cleanup; forcing cleanup. [ 3300.886509] Lustre: fir-OST0035: Not available for connect from 10.49.21.2@o2ib1 (stopping) [ 3300.894862] Lustre: Skipped 144 previous similar messages [ 3301.091687] LustreError: 50827:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.0.10.3@o2ib7 arrived at 1584566721 with bad export cookie 17544123070655633349 [ 3301.896311] Lustre: fir-OST0031: Not available for connect from 10.49.25.18@o2ib1 (stopping) [ 3301.904755] Lustre: Skipped 310 previous similar messages [ 3302.824042] LustreError: 137-5: fir-OST0037_UUID: not available for connect from 10.50.5.15@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3302.841321] LustreError: Skipped 1395 previous similar messages [ 3303.601095] Lustre: server umount fir-OST0037 complete [ 3304.097882] LustreError: 50827:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.0.10.3@o2ib7 arrived at 1584566724 with bad export cookie 17544123070655624207 [ 3304.831086] LustreError: 137-5: fir-OST0037_UUID: not available for connect from 10.49.23.4@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3304.848367] LustreError: Skipped 710 previous similar messages [ 3307.103677] LustreError: 49477:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) ldlm_cancel from 10.0.10.3@o2ib7 arrived at 1584566727 with bad export cookie 17544123070655596669 [ 3307.119136] LustreError: 49477:0:(ldlm_lockd.c:2324:ldlm_cancel_handler()) Skipped 1 previous similar message [ 3307.748476] md3: detected capacity change from 64011422924800 to 0 [ 3307.754670] md: md3 stopped. [ 3307.787895] md9: detected capacity change from 64011422924800 to 0 [ 3307.794090] md: md9 stopped. [ 3308.003999] md11: detected capacity change from 64011422924800 to 0 [ 3308.010283] md: md11 stopped. [ 3308.039883] md5: detected capacity change from 64011422924800 to 0 [ 3308.041654] md7: detected capacity change from 64011422924800 to 0 [ 3308.041663] md: md7 stopped. [ 3308.055140] md: md5 stopped. [ 3308.191139] md1: detected capacity change from 64011422924800 to 0 [ 3308.197336] md: md1 stopped. [ 3308.834380] LustreError: 137-5: fir-OST0039_UUID: not available for connect from 10.50.3.61@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3308.851692] LustreError: Skipped 1473 previous similar messages [ 3308.894661] md: md9 stopped. [ 3309.135912] md: md5 stopped. [ 3309.136039] md: md7 stopped. [ 3309.140474] md: md11 stopped. [ 3309.899918] md: md9 stopped. [ 3310.138425] md: md7 stopped. [ 3316.840620] LustreError: 137-5: fir-OST0039_UUID: not available for connect from 10.49.21.31@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3316.857988] LustreError: Skipped 2145 previous similar messages [ 3333.074057] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.50.13.9@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3333.091338] LustreError: Skipped 2417 previous similar messages [ 3365.082575] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.50.12.2@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [ 3365.099859] LustreError: Skipped 4729 previous similar messages [ 8069.410873] Lustre: fir-OST0036: haven't heard from client 3c0f2777-beb3-4 (at 10.50.1.60@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d149c00, cur 1584571490 expire 1584571340 last 1584571263 [ 8069.430775] Lustre: Skipped 9 previous similar messages [ 9278.304175] LNetError: 47354:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [ 9278.314346] LNetError: 47354:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.3@o2ib7 (105): c: 7, oc: 0, rc: 8 [ 9700.472795] Lustre: fir-OST0030: Connection restored to f82c1c78-5289-8d3f-c213-fe2a9908b1d9 (at 10.0.10.3@o2ib7) [ 9700.483074] Lustre: Skipped 3748 previous similar messages [ 9775.214632] LustreError: 49219:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cc51ef17-5ec1-4 claims 2822144 GRANT, real grant 0 [ 9775.226936] LustreError: 49219:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 4 previous similar messages [ 9938.668373] LustreError: 49327:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 9823016a-d5f7-4 claims 1171456 GRANT, real grant 0 [10073.827282] LustreError: 49356:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 653b182b-aad1-4 claims 2482176 GRANT, real grant 0 [10073.839617] LustreError: 49356:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 2 previous similar messages [10423.112288] LustreError: 49342:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dba6cbec-f37c-4 claims 3399680 GRANT, real grant 0 [11810.400599] Lustre: fir-OST0032: haven't heard from client 15c6dd40-b461-4 (at 10.50.1.11@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e0aa000, cur 1584575231 expire 1584575081 last 1584575004 [11810.420529] Lustre: Skipped 5 previous similar messages [12058.731038] Lustre: fir-OST0030: Connection restored to 15c6dd40-b461-4 (at 10.50.1.11@o2ib2) [12058.739568] Lustre: Skipped 5 previous similar messages [12137.038769] Lustre: 47387:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584575550/real 1584575550] req@ffff9febf060e300 x1661535948825664/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1584575557 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [12137.066813] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [12142.676655] Lustre: fir-OST0038: Client 541f81d4-bd4f-4 (at 10.50.7.3@o2ib2) reconnecting [12142.684869] Lustre: Skipped 7 previous similar messages [12142.690121] Lustre: fir-OST0038: Connection restored to 541f81d4-bd4f-4 (at 10.50.7.3@o2ib2) [12175.756138] LustreError: 137-5: fir-OST0039_UUID: not available for connect from 10.50.7.3@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [12175.773371] LustreError: Skipped 3017 previous similar messages [12238.295479] LNetError: 47354:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [12238.305483] LNetError: 47354:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (11): c: 0, oc: 0, rc: 8 [12238.317692] LNetError: 47364:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.0.10.51@o2ib7 added to recovery queue. Health = 900 [12238.330559] LNetError: 47364:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) Skipped 3 previous similar messages [12238.341366] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12239.295459] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 1 seconds [12239.305566] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12240.295459] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 2 seconds [12240.305544] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 4 previous similar messages [12240.314859] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12242.295445] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 4 seconds [12242.305528] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 3 previous similar messages [12242.314843] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12245.295442] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [12245.305523] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 7 previous similar messages [12247.295448] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12247.307446] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) Skipped 2 previous similar messages [12251.295419] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [12251.305504] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 20 previous similar messages [12256.295416] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12256.307412] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) Skipped 3 previous similar messages [12261.295389] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 8 seconds [12261.305470] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 55 previous similar messages [12267.295410] Lustre: 47388:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1584575653/real 1584575687] req@ffff9febbfbd3a80 x1661535948917888/t0(0) o101->fir-MDT0000-lwp-OST0032@10.0.10.51@o2ib7:23/10 lens 456/496 e 0 to 1 dl 1584575697 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [12267.295422] Lustre: fir-MDT0000-lwp-OST003a: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [12267.295452] LustreError: 47391:0:(qsd_handler.c:340:qsd_req_completion()) $$$ DQACQ failed with -5, flags:0x1 qsd:fir-OST0036 qtype:usr id:324884 enforced:1 granted: 1073741824 pending:0 waiting:0 req:1 usage: 131080 qunit:0 qtune:0 edquot:0 default:yes [12267.295454] LustreError: 47389:0:(qsd_handler.c:340:qsd_req_completion()) $$$ DQACQ failed with -5, flags:0x1 qsd:fir-OST0036 qtype:grp id:24300 enforced:1 granted: 67108864 pending:0 waiting:0 req:1 usage: 60324 qunit:0 qtune:0 edquot:0 default:yes [12267.384704] Lustre: 47388:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [12268.295426] LustreError: 47375:0:(qsd_handler.c:340:qsd_req_completion()) $$$ DQACQ failed with -5, flags:0x1 qsd:fir-OST0032 qtype:grp id:39584 enforced:1 granted: 67108864 pending:0 waiting:0 req:1 usage: 46680 qunit:0 qtune:0 edquot:0 default:yes [12268.295448] Lustre: fir-MDT0000-lwp-OST0038: Connection to fir-MDT0000 (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [12268.295450] Lustre: Skipped 4 previous similar messages [12268.338726] LustreError: 47375:0:(qsd_handler.c:340:qsd_req_completion()) Skipped 6 previous similar messages [12268.348662] Lustre: 47375:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1584575653/real 1584575688] req@ffff9fcf2a1d3180 x1661535948918080/t0(0) o101->fir-MDT0000-lwp-OST003a@10.0.10.51@o2ib7:23/10 lens 456/496 e 0 to 1 dl 1584575697 ref 1 fl Rpc:eXN/0/ffffffff rc 0/-1 [12268.377427] Lustre: 47375:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [12271.295398] Lustre: 47375:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1584575653/real 1584575691] req@ffff9fead34e0900 x1661535948920128/t0(0) o103->fir-MDT0000-lwp-OST0030@10.0.10.51@o2ib7:17/18 lens 328/224 e 0 to 1 dl 1584575697 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [12271.295428] LustreError: 47420:0:(qsd_handler.c:340:qsd_req_completion()) $$$ DQACQ failed with -5, flags:0x1 qsd:fir-OST0032 qtype:grp id:1953 enforced:1 granted: 67108864 pending:0 waiting:0 req:1 usage: 21449856 qunit:0 qtune:0 edquot:0 default:yes [12271.295430] LustreError: 47420:0:(qsd_handler.c:340:qsd_req_completion()) Skipped 7 previous similar messages [12271.356225] Lustre: 47375:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 23 previous similar messages [12273.405408] LustreError: 47422:0:(qsd_handler.c:340:qsd_req_completion()) $$$ DQACQ failed with -5, flags:0x1 qsd:fir-OST003a qtype:prj id:328022 enforced:1 granted: 269738908 pending:0 waiting:0 req:1 usage: 1501280 qunit:0 qtune:0 edquot:0 default:yes [12273.427849] LustreError: 47422:0:(qsd_handler.c:340:qsd_req_completion()) Skipped 15 previous similar messages [12276.295362] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12276.307361] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) Skipped 7 previous similar messages [12276.317382] Lustre: 47382:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1584575653/real 1584575696] req@ffff9febf0961680 x1661535948922688/t0(0) o103->fir-MDT0000-lwp-OST0030@10.0.10.51@o2ib7:17/18 lens 328/224 e 0 to 1 dl 1584575697 ref 1 fl Rpc:eX/0/ffffffff rc 0/-1 [12276.346024] Lustre: 47382:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 32 previous similar messages [12278.295345] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [12278.305429] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 96 previous similar messages [12309.421272] LNetError: 70310:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12309.433273] LNetError: 70310:0:(lib-msg.c:485:lnet_handle_local_failure()) Skipped 7 previous similar messages [12347.522272] Lustre: fir-OST0034: Client 2f3a422b-dba8-4 (at 10.50.0.63@o2ib2) reconnecting [12347.530557] Lustre: fir-OST0034: Connection restored to 2f3a422b-dba8-4 (at 10.50.0.63@o2ib2) [12353.720273] LustreError: 137-5: fir-OST0037_UUID: not available for connect from 10.50.0.63@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [12369.295084] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 5 seconds [12369.305172] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 25 previous similar messages [12373.581083] LNetError: 70176:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12373.593376] LNetError: 70176:0:(lib-msg.c:485:lnet_handle_local_failure()) Skipped 6 previous similar messages [12435.415301] Lustre: fir-OST0038: haven't heard from client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf9f26000, cur 1584575856 expire 1584575706 last 1584575629 [12435.435951] Lustre: Skipped 5 previous similar messages [12453.398041] Lustre: fir-OST0030: haven't heard from client fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80edb800, cur 1584575874 expire 1584575724 last 1584575647 [12496.133656] Lustre: fir-OST0030: Connection restored to fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) [12496.142881] Lustre: Skipped 2 previous similar messages [12511.294657] LNetError: 47354:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 1 seconds [12511.304660] LNetError: 47354:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (42): c: 0, oc: 0, rc: 8 [12511.317043] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [12511.329069] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) Skipped 6 previous similar messages [12610.294367] LNetError: 47354:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: tx_queue, 0 seconds [12610.304372] LNetError: 47354:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.51@o2ib7 (41): c: 0, oc: 0, rc: 8 [12657.401986] Lustre: fir-OST0034: haven't heard from client 48a86d34-282f-4 (at 10.50.5.38@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7fdbf400, cur 1584576078 expire 1584575928 last 1584575851 [12657.421918] Lustre: Skipped 4 previous similar messages [12664.414815] Lustre: fir-OST0036: haven't heard from client 48a86d34-282f-4 (at 10.50.5.38@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d251400, cur 1584576085 expire 1584575935 last 1584575858 [12664.434719] Lustre: Skipped 1 previous similar message [12674.430529] Lustre: fir-OST003a: haven't heard from client 48a86d34-282f-4 (at 10.50.5.38@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7eb14400, cur 1584576095 expire 1584575945 last 1584575868 [12769.980050] LustreError: 167-0: fir-MDT0000-lwp-OST0038: This client was evicted by fir-MDT0000; in progress operations using this service will fail. [12769.993556] LustreError: 47385:0:(client.c:1197:ptlrpc_import_delay_req()) @@@ invalidate in flight req@ffff9febecc3f080 x1661535949333952/t0(0) o103->fir-MDT0000-lwp-OST0036@10.0.10.52@o2ib7:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:W/0/ffffffff rc 0/-1 [12770.022272] Lustre: fir-MDT0000-lwp-OST0038: Connection restored to 10.0.10.52@o2ib7 (at 10.0.10.52@o2ib7) [12770.031930] Lustre: Skipped 7 previous similar messages [12770.232028] LustreError: 11-0: fir-MDT0000-lwp-OST003a: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12770.741582] LustreError: 11-0: fir-MDT0000-lwp-OST0036: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12770.752586] LustreError: Skipped 5 previous similar messages [12775.721057] LustreError: 11-0: fir-MDT0000-lwp-OST0038: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12775.732011] LustreError: Skipped 5 previous similar messages [12779.086037] LustreError: 11-0: fir-MDT0000-lwp-OST0038: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12779.097018] LustreError: Skipped 13 previous similar messages [12786.103132] LustreError: 11-0: fir-MDT0000-lwp-OST0038: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12786.114108] LustreError: Skipped 3 previous similar messages [12797.121021] LustreError: 11-0: fir-MDT0000-lwp-OST0038: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12797.131990] LustreError: Skipped 3 previous similar messages [12821.227089] LustreError: 11-0: fir-MDT0000-lwp-OST0038: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12821.238041] LustreError: Skipped 7 previous similar messages [12845.243929] Lustre: Evicted from MGS (at 10.0.10.51@o2ib7) after server handle changed from 0x22d279060f9ef074 to 0xae6d2c631bd48d67 [12845.256013] Lustre: MGC10.0.10.51@o2ib7: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [12853.247891] LustreError: 11-0: fir-MDT0000-lwp-OST0038: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12853.258858] LustreError: Skipped 51 previous similar messages [12928.577706] LustreError: 11-0: fir-MDT0000-lwp-OST0038: operation quota_acquire to node 10.0.10.52@o2ib7 failed: rc = -11 [12928.588695] LustreError: Skipped 212 previous similar messages [13051.840535] Lustre: fir-OST0032: deleting orphan objects from 0x0:90615136 to 0x0:90615169 [13051.840569] Lustre: fir-OST003a: deleting orphan objects from 0x0:93215766 to 0x0:93215809 [13051.840599] Lustre: fir-OST0038: deleting orphan objects from 0x0:93307087 to 0x0:93307105 [13051.840666] Lustre: fir-OST0036: deleting orphan objects from 0x0:91542330 to 0x0:91542369 [13051.840702] Lustre: fir-OST0034: deleting orphan objects from 0x0:92671751 to 0x0:92671777 [13051.841876] Lustre: fir-OST0030: deleting orphan objects from 0x0:91199004 to 0x0:91199041 [13067.357457] Lustre: fir-OST0030: Connection restored to 48a86d34-282f-4 (at 10.50.5.38@o2ib2) [13067.365988] Lustre: Skipped 5 previous similar messages [13395.992505] LustreError: 49283:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 22edcf71-553a-4 claims 28672 GRANT, real grant 0 [13396.004677] LustreError: 49283:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 4 previous similar messages [13852.424992] LustreError: 49286:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 163840 GRANT, real grant 0 [14244.824694] LustreError: 49192:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 99638e11-6747-4 claims 28672 GRANT, real grant 0 [14244.836800] LustreError: 49192:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 2 previous similar messages [14814.460268] LustreError: 49219:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6dd0a0db-fe9d-4 claims 28672 GRANT, real grant 0 [15326.502006] LustreError: 49272:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 88de242f-0ae5-4 claims 28672 GRANT, real grant 0 [15326.514103] LustreError: 49272:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [16149.285279] LustreError: 49353:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli e7e26766-3907-4 claims 8482816 GRANT, real grant 0 [16149.297541] LustreError: 49353:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [17112.282820] LustreError: 49132:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 15e5b565-193d-4 claims 49152 GRANT, real grant 0 [18147.662952] LustreError: 49073:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 28672 GRANT, real grant 0 [18147.675031] LustreError: 49073:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [18874.477260] LustreError: 49095:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cbf4c94a-83d4-4 claims 32768 GRANT, real grant 0 [18874.489346] LustreError: 49095:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [19541.197248] LustreError: 51030:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli f4b79039-bcb1-4 claims 2482176 GRANT, real grant 0 [19541.209508] LustreError: 51030:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 3 previous similar messages [20322.760231] LustreError: 48900:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli cae111ff-67f4-4 claims 16752640 GRANT, real grant 0 [20322.772605] LustreError: 48900:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [21605.965175] LustreError: 49091:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 814fcb08-b0c5-4 claims 28672 GRANT, real grant 0 [21918.813082] LustreError: 49392:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli b222a878-96be-4 claims 143360 GRANT, real grant 0 [21918.825302] LustreError: 49392:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [22519.169255] LustreError: 49286:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 72c8f4c4-9808-4 claims 28672 GRANT, real grant 0 [23618.733630] perf: interrupt took too long (2504 > 2500), lowering kernel.perf_event_max_sample_rate to 79000 [23651.834980] LustreError: 49129:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 5cfc0423-37f2-4 claims 28672 GRANT, real grant 0 [24737.658650] LustreError: 49334:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 00d873ba-49ea-4 claims 28672 GRANT, real grant 0 [25106.591646] LustreError: 48891:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli fa1e1112-266d-4 claims 4169728 GRANT, real grant 0 [25434.658193] LustreError: 49129:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli bc3a5e42-5951-4 claims 122880 GRANT, real grant 0 [25799.761389] LustreError: 48907:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli be752b6f-e9b6-4 claims 28672 GRANT, real grant 0 [26880.876612] LustreError: 49235:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 49eb42a9-595b-4 claims 2822144 GRANT, real grant 0 [28829.728654] LustreError: 49234:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli dba6cbec-f37c-4 claims 8413184 GRANT, real grant 0 [29562.439631] LustreError: 49330:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli bbc7752c-d751-4 claims 28672 GRANT, real grant 0 [30199.189325] LustreError: 49133:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 4cb83747-634b-4 claims 45056 GRANT, real grant 0 [30451.853341] LustreError: 49272:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli f4b79039-bcb1-4 claims 16752640 GRANT, real grant 0 [30476.808355] LustreError: 49187:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3a5b08eb-e519-4 claims 122880 GRANT, real grant 0 [31424.806077] LustreError: 48892:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli fab83063-bce6-4 claims 16752640 GRANT, real grant 0 [31712.358052] Lustre: fir-OST0030: haven't heard from client ca4d9d7f-c632-4 (at 10.50.9.23@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb800cec00, cur 1584595133 expire 1584594983 last 1584594906 [31712.378015] Lustre: Skipped 2 previous similar messages [31719.357583] Lustre: fir-OST0032: haven't heard from client ca4d9d7f-c632-4 (at 10.50.9.23@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e029400, cur 1584595140 expire 1584594990 last 1584594913 [31725.368430] Lustre: fir-OST0036: haven't heard from client ca4d9d7f-c632-4 (at 10.50.9.23@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d0a5c00, cur 1584595146 expire 1584594996 last 1584594919 [31725.388337] Lustre: Skipped 3 previous similar messages [31793.239548] Lustre: fir-OST0030: Connection restored to ca4d9d7f-c632-4 (at 10.50.9.23@o2ib2) [31793.248080] Lustre: Skipped 4 previous similar messages [32492.355800] Lustre: fir-OST003a: haven't heard from client f4363950-d6c3-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ef72800, cur 1584595913 expire 1584595763 last 1584595686 [32574.249608] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [32574.258133] Lustre: Skipped 5 previous similar messages [33182.886053] LustreError: 48900:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 5682229c-f010-4 claims 28672 GRANT, real grant 0 [33860.523885] LustreError: 49095:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 1a277e81-0585-4 claims 28672 GRANT, real grant 0 [33969.892580] LustreError: 49095:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 240ede69-17ef-4 claims 610304 GRANT, real grant 0 [34059.118129] LustreError: 49342:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 95e66b92-7a58-4 claims 98304 GRANT, real grant 0 [34213.057805] LustreError: 49315:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dba6cbec-f37c-4 claims 16752640 GRANT, real grant 3399680 [36134.334662] LustreError: 49274:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 7113d51c-71d5-4 claims 28672 GRANT, real grant 0 [36832.387254] LustreError: 49338:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 15e5b565-193d-4 claims 28672 GRANT, real grant 0 [36983.226543] LustreError: 49132:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli f44d5096-53d1-4 claims 28672 GRANT, real grant 0 [37307.306559] LustreError: 49373:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a65d538b-3161-4 claims 28672 GRANT, real grant 0 [37721.497838] LustreError: 49342:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli a65d538b-3161-4 claims 28672 GRANT, real grant 0 [37852.999866] LustreError: 49235:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 7822258b-c265-4 claims 28672 GRANT, real grant 0 [37993.877480] LustreError: 49307:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 9bb420b9-4b7e-4 claims 2822144 GRANT, real grant 0 [38389.143265] LustreError: 49339:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 7822258b-c265-4 claims 28672 GRANT, real grant 0 [38751.344400] Lustre: fir-OST0034: haven't heard from client eceee209-ec05-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7fc11400, cur 1584602172 expire 1584602022 last 1584601945 [38751.364332] Lustre: Skipped 5 previous similar messages [38763.365485] Lustre: fir-OST0032: haven't heard from client eceee209-ec05-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e053c00, cur 1584602184 expire 1584602034 last 1584601957 [38763.385426] Lustre: Skipped 4 previous similar messages [38852.291316] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [38852.299849] Lustre: Skipped 5 previous similar messages [42198.599032] LustreError: 49228:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli bc12991d-b630-4 claims 16752640 GRANT, real grant 0 [42230.397254] LustreError: 49296:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli dff4fc43-431c-4 claims 286720 GRANT, real grant 0 [42525.076264] LustreError: 49314:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli bc12991d-b630-4 claims 282624 GRANT, real grant 0 [47029.119786] LustreError: 49307:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 37a9513e-0a78-4 claims 28672 GRANT, real grant 0 [54651.595922] LustreError: 49148:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 05bde955-e83a-4 claims 16752640 GRANT, real grant 0 [54744.087663] LustreError: 49003:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 0ce69c96-29d6-4 claims 589824 GRANT, real grant 0 [55680.278630] LustreError: 49148:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli dba6cbec-f37c-4 claims 4169728 GRANT, real grant 0 [56777.439409] LustreError: 49311:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 665344bc-9abf-4 claims 28672 GRANT, real grant 0 [56941.318391] Lustre: fir-OST0036: haven't heard from client 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d026400, cur 1584620362 expire 1584620212 last 1584620135 [57021.261889] Lustre: fir-OST0030: Connection restored to 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) [57021.270419] Lustre: Skipped 5 previous similar messages [59502.845870] LustreError: 49240:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 8db8300c-7159-4 claims 7655424 GRANT, real grant 0 [60257.395653] perf: interrupt took too long (3132 > 3130), lowering kernel.perf_event_max_sample_rate to 63000 [64466.133423] LustreError: 48931:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli f44d5096-53d1-4 claims 11526144 GRANT, real grant 0 [67662.544289] LustreError: 49009:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 5b050ff2-eda0-4 claims 28672 GRANT, real grant 0 [68307.084132] LustreError: 49268:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 275b0d00-9e41-4 claims 28672 GRANT, real grant 0 [69808.009225] LustreError: 49309:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli eac7d7e6-7a39-4 claims 536576 GRANT, real grant 0 [72311.692285] LustreError: 49353:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a61ba52f-0e9d-4 claims 28672 GRANT, real grant 0 [72595.696020] LustreError: 49271:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a9a870dd-42ae-4 claims 974848 GRANT, real grant 0 [72732.326812] LustreError: 49388:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 1d40ab79-d6a8-4 claims 2895872 GRANT, real grant 0 [73023.740850] LustreError: 49204:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 5616204e-fc92-4 claims 851968 GRANT, real grant 0 [75326.749973] LustreError: 49069:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli daeb2854-fef7-4 claims 28672 GRANT, real grant 0 [75329.226480] LustreError: 49071:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli e19e4c6e-1211-4 claims 368640 GRANT, real grant 0 [75359.363007] LustreError: 49311:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli aa070619-7399-4 claims 28672 GRANT, real grant 0 [75449.970689] LustreError: 49356:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli aa070619-7399-4 claims 57344 GRANT, real grant 0 [75512.282686] Lustre: fir-OST0034: haven't heard from client a2a51302-74aa-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb6f11d800, cur 1584638933 expire 1584638783 last 1584638706 [75512.302573] Lustre: Skipped 5 previous similar messages [75591.969906] Lustre: fir-OST0030: Connection restored to 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) [75591.978430] Lustre: Skipped 5 previous similar messages [75640.218416] LustreError: 49373:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli b73e1dab-2ad6-4 claims 483328 GRANT, real grant 0 [75650.297447] LustreError: 49296:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli b73e1dab-2ad6-4 claims 28672 GRANT, real grant 0 [75650.309534] LustreError: 49296:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [76426.237914] LustreError: 49356:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 27b32a6c-5b01-4 claims 28672 GRANT, real grant 0 [78774.024911] LustreError: 49123:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 35cd2c2f-e02c-4 claims 28672 GRANT, real grant 0 [79093.381676] LustreError: 48896:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 767f4960-85ab-4 claims 1441792 GRANT, real grant 0 [80234.270595] Lustre: fir-OST0030: haven't heard from client e584cba3-332a-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f5800, cur 1584643655 expire 1584643505 last 1584643428 [80234.290504] Lustre: Skipped 5 previous similar messages [80310.148308] Lustre: fir-OST0030: Connection restored to 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) [80310.156854] Lustre: Skipped 5 previous similar messages [80446.015213] LustreError: 49137:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 39b158e5-2e75-4 claims 1171456 GRANT, real grant 0 [82993.638678] LustreError: 49244:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2b06b865-da7d-4 claims 323584 GRANT, real grant 0 [83069.438604] LustreError: 49256:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 6e05ab9f-8c33-4 claims 77824 GRANT, real grant 0 [83386.714023] LustreError: 49229:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 1f9d385a-f33f-4 claims 81920 GRANT, real grant 0 [83503.524832] LustreError: 49334:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 95eea94a-c8bd-4 claims 36864 GRANT, real grant 0 [84720.320977] LustreError: 49075:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 17f990d4-4bb0-4 claims 434176 GRANT, real grant 0 [88615.075250] LustreError: 49137:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli dbc3d438-69dd-4 claims 4100096 GRANT, real grant 0 [88664.934790] LustreError: 49336:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dd1fe72f-e583-4 claims 77824 GRANT, real grant 0 [89702.539990] LustreError: 49388:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 65e9680f-4e61-4 claims 2482176 GRANT, real grant 0 [90529.458666] LustreError: 67480:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 2147e45e-5aee-4 claims 28672 GRANT, real grant 0 [98589.690605] LustreError: 67482:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a1b5a428-453b-4 claims 16752640 GRANT, real grant 0 [99203.878349] LustreError: 49299:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli bdf290df-3ae9-4 claims 16752640 GRANT, real grant 0 [106105.306902] LustreError: 49137:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 6e05ab9f-8c33-4 claims 81920 GRANT, real grant 0 [106193.333221] LustreError: 49237:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli be752b6f-e9b6-4 claims 94208 GRANT, real grant 28672 [108534.604797] LustreError: 49134:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 6d03d9a7-b1f2-4 claims 28672 GRANT, real grant 0 [111068.355634] LustreError: 49229:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 95038ffe-45ab-4 claims 28672 GRANT, real grant 0 [111430.015265] LustreError: 49341:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 9bb420b9-4b7e-4 claims 28672 GRANT, real grant 0 [113066.924412] LustreError: 48329:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli aa070619-7399-4 claims 81920 GRANT, real grant 0 [113202.577129] LustreError: 49334:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli c7de84cf-8a38-4 claims 802816 GRANT, real grant 0 [114176.516872] LustreError: 48979:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 67876c18-c2e3-4 claims 90112 GRANT, real grant 0 [114796.034341] LustreError: 49315:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a1986753-a6b4-4 claims 802816 GRANT, real grant 0 [115081.271748] LustreError: 49256:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6deab853-679e-4 claims 229376 GRANT, real grant 0 [115428.887099] LustreError: 49127:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli e0e2514e-9993-4 claims 28672 GRANT, real grant 0 [116878.847141] LustreError: 49300:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 22edcf71-553a-4 claims 16752640 GRANT, real grant 28672 [118098.945179] LustreError: 49133:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 77a9a84e-c513-4 claims 28672 GRANT, real grant 0 [119395.198373] Lustre: fir-OST0036: haven't heard from client 4f32e72b-02b6-4 (at 10.50.9.6@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d025000, cur 1584682816 expire 1584682666 last 1584682589 [119395.218296] Lustre: Skipped 5 previous similar messages [119419.957111] Lustre: fir-OST0030: Connection restored to 4f32e72b-02b6-4 (at 10.50.9.6@o2ib2) [119419.965635] Lustre: Skipped 5 previous similar messages [121885.193071] Lustre: fir-OST0030: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80312800, cur 1584685306 expire 1584685156 last 1584685079 [121885.213035] Lustre: Skipped 5 previous similar messages [122384.198185] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [122413.269978] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [122420.294022] Lustre: fir-OST0030: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [122611.194489] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eaa83000, cur 1584686032 expire 1584685882 last 1584685805 [122611.214388] Lustre: Skipped 5 previous similar messages [122613.669705] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [122640.206778] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd1292000, cur 1584686061 expire 1584685911 last 1584685834 [122647.192705] Lustre: fir-OST0030: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938e9d400, cur 1584686068 expire 1584685918 last 1584685841 [122786.053401] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [122823.109414] Lustre: fir-OST003a: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [122840.211952] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931931000, cur 1584686261 expire 1584686111 last 1584686034 [123012.194044] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7dce5400, cur 1584686433 expire 1584686283 last 1584686206 [123049.212463] Lustre: fir-OST003a: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930322c00, cur 1584686470 expire 1584686320 last 1584686243 [123485.808931] LustreError: 67481:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3a5b08eb-e519-4 claims 450560 GRANT, real grant 122880 [123583.684134] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [123613.748102] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [123620.870313] Lustre: fir-OST0030: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [123810.224345] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d4000, cur 1584687231 expire 1584687081 last 1584687004 [123814.179798] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [123840.199964] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931937000, cur 1584687261 expire 1584687111 last 1584687034 [123847.192688] Lustre: fir-OST0030: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb9d3000, cur 1584687268 expire 1584687118 last 1584687041 [123985.539632] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [124022.595544] Lustre: fir-OST003a: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [124040.197119] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf721e400, cur 1584687461 expire 1584687311 last 1584687234 [124212.194677] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf902b000, cur 1584687633 expire 1584687483 last 1584687406 [124249.190702] Lustre: fir-OST003a: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f5a8800, cur 1584687670 expire 1584687520 last 1584687443 [124559.055168] LustreError: 49338:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 5b88c4a2-39df-4 claims 4169728 GRANT, real grant 0 [124784.194389] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [124813.250329] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [125011.188365] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd0bec000, cur 1584688432 expire 1584688282 last 1584688205 [125013.666308] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [125013.674843] Lustre: Skipped 1 previous similar message [125040.208385] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf91c3800, cur 1584688461 expire 1584688311 last 1584688234 [125185.921693] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [125240.187959] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef50e400, cur 1584688661 expire 1584688511 last 1584688434 [125240.207845] Lustre: Skipped 1 previous similar message [125412.190694] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b8600a400, cur 1584688833 expire 1584688683 last 1584688606 [125546.801898] LustreError: 49318:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 732c4259-5045-4 claims 6176768 GRANT, real grant 0 [125983.680812] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [125983.689344] Lustre: Skipped 1 previous similar message [126013.744730] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [126210.187644] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930326800, cur 1584689631 expire 1584689481 last 1584689404 [126210.207563] Lustre: Skipped 1 previous similar message [126214.176467] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [126214.185003] Lustre: Skipped 1 previous similar message [126240.243949] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93132ec00, cur 1584689661 expire 1584689511 last 1584689434 [126385.536237] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [126440.198854] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93132dc00, cur 1584689861 expire 1584689711 last 1584689634 [126440.218761] Lustre: Skipped 1 previous similar message [126604.081062] LustreError: 49256:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 0271c837-bb5b-4 claims 32768 GRANT, real grant 0 [126612.200538] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a5800, cur 1584690033 expire 1584689883 last 1584689806 [126749.268929] LustreError: 49127:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 95e66b92-7a58-4 claims 16752640 GRANT, real grant 98304 [127075.445755] LustreError: 49309:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli f7ce444e-0508-4 claims 16752640 GRANT, real grant 0 [127183.295188] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [127183.303717] Lustre: Skipped 1 previous similar message [127202.636042] LustreError: 91448:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a9a870dd-42ae-4 claims 4169728 GRANT, real grant 1003520 [127213.359037] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [127410.203029] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80e400, cur 1584690831 expire 1584690681 last 1584690604 [127410.222970] Lustre: Skipped 1 previous similar message [127413.794555] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [127413.803086] Lustre: Skipped 1 previous similar message [127440.188776] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938e9cc00, cur 1584690861 expire 1584690711 last 1584690634 [127468.443219] LustreError: 49264:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 97171295-2f64-4 claims 356352 GRANT, real grant 0 [127586.174443] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [127640.215489] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf91c1c00, cur 1584691061 expire 1584690911 last 1584690834 [127640.235377] Lustre: Skipped 1 previous similar message [127812.190933] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef15c00, cur 1584691233 expire 1584691083 last 1584691006 [128383.933332] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [128383.941877] Lustre: Skipped 1 previous similar message [128413.997169] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [128610.209586] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf91a9c00, cur 1584692031 expire 1584691881 last 1584691804 [128610.229466] Lustre: Skipped 1 previous similar message [128613.405002] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [128613.413538] Lustre: Skipped 1 previous similar message [128640.188378] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9386d2c00, cur 1584692061 expire 1584691911 last 1584691834 [128785.788692] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [128840.186187] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5f61000, cur 1584692261 expire 1584692111 last 1584692034 [128840.206100] Lustre: Skipped 1 previous similar message [129012.202111] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd939e74400, cur 1584692433 expire 1584692283 last 1584692206 [129462.422093] LustreError: 49380:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 6470996b-3104-4 claims 28672 GRANT, real grant 0 [129583.419407] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [129583.427933] Lustre: Skipped 1 previous similar message [129613.483521] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [129617.634267] LustreError: 49265:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli bc3a5e42-5951-4 claims 6135808 GRANT, real grant 151552 [129627.954270] LustreError: 48326:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a61ba52f-0e9d-4 claims 32768 GRANT, real grant 28672 [129810.197280] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b9e7dc000, cur 1584693231 expire 1584693081 last 1584693004 [129810.217190] Lustre: Skipped 1 previous similar message [129813.851115] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [129813.859717] Lustre: Skipped 1 previous similar message [129840.182064] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfe450400, cur 1584693261 expire 1584693111 last 1584693034 [129935.720788] LustreError: 49323:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 24485f25-7c65-4 claims 5038080 GRANT, real grant 0 [129986.170740] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [130040.182419] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934ffc00, cur 1584693461 expire 1584693311 last 1584693234 [130040.202316] Lustre: Skipped 1 previous similar message [130212.181403] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf902ec00, cur 1584693633 expire 1584693483 last 1584693406 [130389.547711] LustreError: 49258:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a1986753-a6b4-4 claims 16752640 GRANT, real grant 802816 [130771.006344] LustreError: 49324:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 00c7f158-cc8b-4 claims 16695296 GRANT, real grant 24576 [130783.801437] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [130783.809968] Lustre: Skipped 1 previous similar message [130813.865450] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [131010.184935] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd829f000, cur 1584694431 expire 1584694281 last 1584694204 [131010.204854] Lustre: Skipped 1 previous similar message [131013.241133] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [131013.249662] Lustre: Skipped 1 previous similar message [131040.183732] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad921000, cur 1584694461 expire 1584694311 last 1584694234 [131185.529804] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [131240.187203] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf91ab800, cur 1584694661 expire 1584694511 last 1584694434 [131240.207101] Lustre: Skipped 1 previous similar message [131359.310785] LustreError: 49264:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 5b050ff2-eda0-4 claims 1441792 GRANT, real grant 28672 [131412.178755] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbce685000, cur 1584694833 expire 1584694683 last 1584694606 [131984.184523] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [131984.193055] Lustre: Skipped 1 previous similar message [132013.255491] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [132211.178626] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd3b63800, cur 1584695632 expire 1584695482 last 1584695405 [132211.198528] Lustre: Skipped 1 previous similar message [132213.687103] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [132213.695636] Lustre: Skipped 1 previous similar message [132220.911735] LustreError: 49245:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 4cb83747-634b-4 claims 147456 GRANT, real grant 0 [132240.180889] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbba7a4000, cur 1584695661 expire 1584695511 last 1584695434 [132386.038759] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [132440.178018] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdb09902400, cur 1584695861 expire 1584695711 last 1584695634 [132440.197940] Lustre: Skipped 1 previous similar message [132572.100999] LustreError: 91448:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 240ede69-17ef-4 claims 4218880 GRANT, real grant 610304 [132612.182944] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bce6d0400, cur 1584696033 expire 1584695883 last 1584695806 [133183.670418] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [133183.678952] Lustre: Skipped 1 previous similar message [133213.733520] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [133410.176736] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb6f118c00, cur 1584696831 expire 1584696681 last 1584696604 [133410.196623] Lustre: Skipped 1 previous similar message [133414.133094] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [133414.141624] Lustre: Skipped 1 previous similar message [133440.197108] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9513400, cur 1584696861 expire 1584696711 last 1584696634 [133585.396804] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [133640.182062] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf433c800, cur 1584697061 expire 1584696911 last 1584696834 [133640.201959] Lustre: Skipped 1 previous similar message [133704.501624] LustreError: 49248:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 37a9513e-0a78-4 claims 73728 GRANT, real grant 28672 [133757.287553] LustreError: 49291:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cc51ef17-5ec1-4 claims 16752640 GRANT, real grant 2822144 [133781.642997] LustreError: 67499:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 10031104 GRANT, real grant 0 [133812.213418] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf433c000, cur 1584697233 expire 1584697083 last 1584697006 [133879.668192] LustreError: 49254:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli f7ce444e-0508-4 claims 16752640 GRANT, real grant 0 [134046.266599] LustreError: 49327:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 0271c837-bb5b-4 claims 1441792 GRANT, real grant 0 [134384.179356] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [134384.187883] Lustre: Skipped 1 previous similar message [134413.267422] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [134611.199833] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd3a2c400, cur 1584698032 expire 1584697882 last 1584697805 [134611.219775] Lustre: Skipped 1 previous similar message [134613.683028] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [134613.691560] Lustre: Skipped 1 previous similar message [134640.176126] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd935e77400, cur 1584698061 expire 1584697911 last 1584697834 [134786.034606] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [134840.175689] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9510400, cur 1584698261 expire 1584698111 last 1584698034 [134840.195589] Lustre: Skipped 1 previous similar message [135012.179073] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd935e77400, cur 1584698433 expire 1584698283 last 1584698206 [135583.793212] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [135583.801739] Lustre: Skipped 1 previous similar message [135613.857220] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [135810.174025] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd3b64000, cur 1584699231 expire 1584699081 last 1584699004 [135810.193906] Lustre: Skipped 1 previous similar message [135813.264855] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [135813.273388] Lustre: Skipped 1 previous similar message [135840.175419] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf3244400, cur 1584699261 expire 1584699111 last 1584699034 [135985.649576] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [136040.179301] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00befa7a400, cur 1584699461 expire 1584699311 last 1584699234 [136040.199229] Lustre: Skipped 1 previous similar message [136212.204808] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbeae4f800, cur 1584699633 expire 1584699483 last 1584699406 [136261.232311] LustreError: 49355:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli dba6cbec-f37c-4 claims 40960 GRANT, real grant 0 [136266.951955] LustreError: 49197:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli ee4b308c-5261-4 claims 16752640 GRANT, real grant 0 [136783.407079] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [136783.415617] Lustre: Skipped 1 previous similar message [136813.471083] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [137010.170415] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9b9e000, cur 1584700431 expire 1584700281 last 1584700204 [137010.190315] Lustre: Skipped 1 previous similar message [137013.854683] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [137013.863218] Lustre: Skipped 1 previous similar message [137040.171742] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7a3bfc00, cur 1584700461 expire 1584700311 last 1584700234 [137078.887459] LustreError: 49267:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6deab853-679e-4 claims 4169728 GRANT, real grant 258048 [137080.849269] LustreError: 49357:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 9891086f-7733-4 claims 3350528 GRANT, real grant 28672 [137186.158300] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [137240.170871] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf721bc00, cur 1584700661 expire 1584700511 last 1584700434 [137240.190748] Lustre: Skipped 1 previous similar message [137412.198435] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ebaaa800, cur 1584700833 expire 1584700683 last 1584700606 [137422.794513] LustreError: 48964:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli c742d3fe-d253-4 claims 1253376 GRANT, real grant 0 [137983.789054] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [137983.797580] Lustre: Skipped 1 previous similar message [138013.852893] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [138210.174257] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf91aac00, cur 1584701631 expire 1584701481 last 1584701404 [138210.194132] Lustre: Skipped 1 previous similar message [138213.228568] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [138213.237103] Lustre: Skipped 1 previous similar message [138240.170648] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b8600dc00, cur 1584701661 expire 1584701511 last 1584701434 [138325.963171] LustreError: 48335:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli aa070619-7399-4 claims 16752640 GRANT, real grant 86016 [138385.516382] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [138440.170857] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5aec00, cur 1584701861 expire 1584701711 last 1584701634 [138440.190772] Lustre: Skipped 1 previous similar message [138612.169165] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb9d6c00, cur 1584702033 expire 1584701883 last 1584701806 [139184.170945] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [139184.179482] Lustre: Skipped 1 previous similar message [139213.226937] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [139341.848238] LustreError: 49235:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli dba6cbec-f37c-4 claims 4169728 GRANT, real grant 40960 [139411.175723] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c5c00, cur 1584702832 expire 1584702682 last 1584702605 [139411.195655] Lustre: Skipped 1 previous similar message [139413.578584] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [139413.587108] Lustre: Skipped 1 previous similar message [139440.166740] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c5800, cur 1584702861 expire 1584702711 last 1584702634 [139585.898312] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [139640.166475] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef13400, cur 1584703061 expire 1584702911 last 1584702834 [139640.186381] Lustre: Skipped 1 previous similar message [139812.168773] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93854d800, cur 1584703233 expire 1584703083 last 1584703006 [140129.450190] LustreError: 102164:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli dff4fc43-431c-4 claims 4169728 GRANT, real grant 1073152 [140383.528991] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [140383.537524] Lustre: Skipped 1 previous similar message [140413.578076] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [140610.166841] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf29bec00, cur 1584704031 expire 1584703881 last 1584703804 [140610.186739] Lustre: Skipped 1 previous similar message [140613.960905] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [140613.969437] Lustre: Skipped 1 previous similar message [140640.166558] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd3a2bc00, cur 1584704061 expire 1584703911 last 1584703834 [140785.320371] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [140840.189298] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bee9cf800, cur 1584704261 expire 1584704111 last 1584704034 [140840.209187] Lustre: Skipped 1 previous similar message [141012.177641] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdb09907400, cur 1584704433 expire 1584704283 last 1584704206 [141583.911145] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [141583.919674] Lustre: Skipped 1 previous similar message [141613.959027] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [141810.163614] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931937000, cur 1584705231 expire 1584705081 last 1584705004 [141810.183493] Lustre: Skipped 1 previous similar message [141813.382801] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [141813.391327] Lustre: Skipped 1 previous similar message [141840.182093] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d6400, cur 1584705261 expire 1584705111 last 1584705034 [141985.830519] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [142040.164860] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bfa62e400, cur 1584705461 expire 1584705311 last 1584705234 [142040.184784] Lustre: Skipped 1 previous similar message [142212.165781] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfe455c00, cur 1584705633 expire 1584705483 last 1584705406 [142783.525297] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [142783.533844] Lustre: Skipped 1 previous similar message [142813.573259] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [143010.167317] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd3b66800, cur 1584706431 expire 1584706281 last 1584706204 [143010.187245] Lustre: Skipped 1 previous similar message [143013.956981] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [143013.965509] Lustre: Skipped 1 previous similar message [143019.504286] LustreError: 49230:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 380928 GRANT, real grant 163840 [143040.174341] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bfa628c00, cur 1584706461 expire 1584706311 last 1584706234 [143185.317131] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [143240.170397] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e52d2400, cur 1584706661 expire 1584706511 last 1584706434 [143240.190288] Lustre: Skipped 1 previous similar message [143412.168715] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf3240c00, cur 1584706833 expire 1584706683 last 1584706606 [143983.907445] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [143983.915970] Lustre: Skipped 1 previous similar message [144013.955480] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [144210.166227] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd8298c00, cur 1584707631 expire 1584707481 last 1584707404 [144210.186171] Lustre: Skipped 1 previous similar message [144213.315099] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [144213.323666] Lustre: Skipped 1 previous similar message [144240.173606] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b5a800, cur 1584707661 expire 1584707511 last 1584707434 [144385.698822] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [144440.169561] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf7fec800, cur 1584707861 expire 1584707711 last 1584707634 [144440.189478] Lustre: Skipped 1 previous similar message [144612.161876] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdb16aeb400, cur 1584708033 expire 1584707883 last 1584707806 [144723.983416] LustreError: 49324:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 16365f08-4eb8-4 claims 28672 GRANT, real grant 0 [145183.457601] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [145183.466147] Lustre: Skipped 1 previous similar message [145213.505584] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [145410.172868] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd1291c00, cur 1584708831 expire 1584708681 last 1584708604 [145410.192774] Lustre: Skipped 1 previous similar message [145413.889300] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [145413.897824] Lustre: Skipped 1 previous similar message [145440.163736] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd1295c00, cur 1584708861 expire 1584708711 last 1584708634 [145585.217036] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [145603.204598] LustreError: 49009:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli aa070619-7399-4 claims 16752640 GRANT, real grant 28672 [145640.159930] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7b497400, cur 1584709061 expire 1584708911 last 1584708834 [145640.179813] Lustre: Skipped 1 previous similar message [145812.186368] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd939e70400, cur 1584709233 expire 1584709083 last 1584709006 [145898.791643] LustreError: 49373:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3e4eec45-c061-4 claims 221184 GRANT, real grant 28672 [145966.356799] LustreError: 49284:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 86db2c50-4e1f-4 claims 45056 GRANT, real grant 0 [146383.967786] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [146383.976312] Lustre: Skipped 1 previous similar message [146414.015789] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [146610.160632] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf5fe8000, cur 1584710031 expire 1584709881 last 1584709804 [146610.180576] Lustre: Skipped 1 previous similar message [146613.375500] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [146613.384029] Lustre: Skipped 1 previous similar message [146640.160403] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf7feb400, cur 1584710061 expire 1584709911 last 1584709834 [146785.823261] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [146840.158427] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef50d400, cur 1584710261 expire 1584710111 last 1584710034 [146840.178347] Lustre: Skipped 1 previous similar message [147012.162674] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef14800, cur 1584710433 expire 1584710283 last 1584710206 [147583.582142] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [147583.590673] Lustre: Skipped 1 previous similar message [147613.630025] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [147810.159234] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febd53f1800, cur 1584711231 expire 1584711081 last 1584711004 [147810.179133] Lustre: Skipped 1 previous similar message [147814.013601] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [147814.022135] Lustre: Skipped 1 previous similar message [147840.178518] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbec5fac00, cur 1584711261 expire 1584711111 last 1584711034 [147985.437363] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [148040.158654] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5e74000, cur 1584711461 expire 1584711311 last 1584711234 [148040.178551] Lustre: Skipped 1 previous similar message [148212.250850] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9b98c00, cur 1584711633 expire 1584711483 last 1584711406 [148353.423836] LustreError: 49324:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dcb1fce2-5124-4 claims 11210752 GRANT, real grant 0 [148454.704505] LustreError: 48931:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli eac7d7e6-7a39-4 claims 8413184 GRANT, real grant 536576 [148783.259941] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [148783.268507] Lustre: Skipped 1 previous similar message [148813.307769] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [148839.175377] LustreError: 49319:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli aa070619-7399-4 claims 4218880 GRANT, real grant 81920 [149010.165174] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb77945c00, cur 1584712431 expire 1584712281 last 1584712204 [149010.185054] Lustre: Skipped 1 previous similar message [149013.691422] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [149013.699958] Lustre: Skipped 1 previous similar message [149040.159382] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9374a4c00, cur 1584712461 expire 1584712311 last 1584712234 [149104.764322] LustreError: 49256:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli c54d471d-70f7-4 claims 155648 GRANT, real grant 28672 [149186.075128] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [149190.313907] LustreError: 50850:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 95038ffe-45ab-4 claims 40960 GRANT, real grant 28672 [149240.162399] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd935e71000, cur 1584712661 expire 1584712511 last 1584712434 [149240.182306] Lustre: Skipped 1 previous similar message [149290.820078] LustreError: 49314:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 23b73773-98ec-4 claims 147456 GRANT, real grant 57344 [149412.158089] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93854c000, cur 1584712833 expire 1584712683 last 1584712606 [149566.440987] LustreError: 49330:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 741376 GRANT, real grant 380928 [149983.833588] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [149983.842122] Lustre: Skipped 1 previous similar message [150013.881453] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [150196.639038] LustreError: 49071:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3a5b08eb-e519-4 claims 557056 GRANT, real grant 450560 [150210.161970] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f576400, cur 1584713631 expire 1584713481 last 1584713404 [150210.181858] Lustre: Skipped 1 previous similar message [150213.305078] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [150213.313611] Lustre: Skipped 1 previous similar message [150240.157101] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a7000, cur 1584713661 expire 1584713511 last 1584713434 [150314.119561] LustreError: 49355:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 4cb83747-634b-4 claims 151552 GRANT, real grant 147456 [150385.688782] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [150440.166493] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef809000, cur 1584713861 expire 1584713711 last 1584713634 [150440.186418] Lustre: Skipped 1 previous similar message [150458.183541] LustreError: 49332:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli fab83063-bce6-4 claims 151552 GRANT, real grant 0 [150612.179405] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf721e400, cur 1584714033 expire 1584713883 last 1584713806 [151183.447278] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [151183.455815] Lustre: Skipped 1 previous similar message [151213.495147] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [151410.154721] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd3b61400, cur 1584714831 expire 1584714681 last 1584714604 [151410.174629] Lustre: Skipped 1 previous similar message [151413.942675] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [151413.951207] Lustre: Skipped 1 previous similar message [151440.160803] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffca9b42000, cur 1584714861 expire 1584714711 last 1584714634 [151585.302336] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [151640.156368] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbecd82400, cur 1584715061 expire 1584714911 last 1584714834 [151640.176255] Lustre: Skipped 1 previous similar message [151701.944901] LustreError: 49092:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cbf4c94a-83d4-4 claims 4014080 GRANT, real grant 32768 [151812.155625] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf9f0cc00, cur 1584715233 expire 1584715083 last 1584715006 [152336.341468] LustreError: 49128:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli b6453fa2-7e92-4 claims 32768 GRANT, real grant 0 [152383.956748] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [152383.965296] Lustre: Skipped 1 previous similar message [152414.004667] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [152538.972978] LustreError: 49071:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 23b73773-98ec-4 claims 155648 GRANT, real grant 147456 [152610.188233] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb79264c00, cur 1584716031 expire 1584715881 last 1584715804 [152610.208154] Lustre: Skipped 1 previous similar message [152613.428237] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [152613.436769] Lustre: Skipped 1 previous similar message [152640.185594] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bce6d6c00, cur 1584716061 expire 1584715911 last 1584715834 [152688.484668] LustreError: 48907:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli ee09de85-2d60-4 claims 319488 GRANT, real grant 57344 [152785.812019] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [152840.180735] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf0572800, cur 1584716261 expire 1584716111 last 1584716034 [152840.200641] Lustre: Skipped 1 previous similar message [153012.186387] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d5800, cur 1584716433 expire 1584716283 last 1584716206 [153583.442431] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [153583.450959] Lustre: Skipped 1 previous similar message [153613.490338] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [153810.154213] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930bc1800, cur 1584717231 expire 1584717081 last 1584717004 [153810.174156] Lustre: Skipped 1 previous similar message [153813.937890] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [153813.946420] Lustre: Skipped 1 previous similar message [153840.176875] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930bc2400, cur 1584717261 expire 1584717111 last 1584717034 [153985.361470] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [154040.152690] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80c400, cur 1584717461 expire 1584717311 last 1584717234 [154040.172573] Lustre: Skipped 1 previous similar message [154212.150694] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5e73000, cur 1584717633 expire 1584717483 last 1584717406 [154784.079860] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [154784.088391] Lustre: Skipped 1 previous similar message [154814.128236] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [155010.147569] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5327800, cur 1584718431 expire 1584718281 last 1584718204 [155010.167452] Lustre: Skipped 1 previous similar message [155013.551318] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [155013.559844] Lustre: Skipped 1 previous similar message [155040.144818] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd1292000, cur 1584718461 expire 1584718311 last 1584718234 [155185.998918] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [155240.155154] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef10000, cur 1584718661 expire 1584718511 last 1584718434 [155240.175035] Lustre: Skipped 1 previous similar message [155406.109989] LustreError: 49323:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli c742d3fe-d253-4 claims 2482176 GRANT, real grant 1253376 [155406.122924] LustreError: 49323:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [155412.163608] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b5f800, cur 1584718833 expire 1584718683 last 1584718606 [155613.850814] LustreError: 49310:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 2147e45e-5aee-4 claims 16752640 GRANT, real grant 15761408 [155983.693334] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [155983.701860] Lustre: Skipped 1 previous similar message [156013.741221] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [156210.154004] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bcef7dc00, cur 1584719631 expire 1584719481 last 1584719404 [156210.173908] Lustre: Skipped 1 previous similar message [156214.124842] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [156214.133369] Lustre: Skipped 1 previous similar message [156240.160961] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a3400, cur 1584719661 expire 1584719511 last 1584719434 [156385.484575] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [156440.156652] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffca9b45c00, cur 1584719861 expire 1584719711 last 1584719634 [156440.176556] Lustre: Skipped 1 previous similar message [156612.144324] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9386d2800, cur 1584720033 expire 1584719883 last 1584719806 [157183.371298] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [157183.379831] Lustre: Skipped 1 previous similar message [157213.419275] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [157410.165257] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd939a65c00, cur 1584720831 expire 1584720681 last 1584720604 [157410.185160] Lustre: Skipped 1 previous similar message [157413.866990] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [157413.875522] Lustre: Skipped 1 previous similar message [157440.175655] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd0beec00, cur 1584720861 expire 1584720711 last 1584720634 [157585.258880] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [157640.146317] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f5af000, cur 1584721061 expire 1584720911 last 1584720834 [157640.166220] Lustre: Skipped 1 previous similar message [157812.140796] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930119c00, cur 1584721233 expire 1584721083 last 1584721006 [158384.009860] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [158384.018384] Lustre: Skipped 1 previous similar message [158414.057758] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [158610.146801] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bcef7d400, cur 1584722031 expire 1584721881 last 1584721804 [158610.166692] Lustre: Skipped 1 previous similar message [158613.481518] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [158613.490063] Lustre: Skipped 1 previous similar message [158640.143221] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9386d7000, cur 1584722061 expire 1584721911 last 1584721834 [158785.833207] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [158840.141762] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7b496800, cur 1584722261 expire 1584722111 last 1584722034 [158840.161646] Lustre: Skipped 1 previous similar message [159012.144341] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbef4ca400, cur 1584722433 expire 1584722283 last 1584722206 [159583.623491] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [159583.632017] Lustre: Skipped 1 previous similar message [159613.671467] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [159810.137961] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5324000, cur 1584723231 expire 1584723081 last 1584723004 [159810.157949] Lustre: Skipped 1 previous similar message [159814.118961] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [159814.127493] Lustre: Skipped 1 previous similar message [159840.145494] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931936800, cur 1584723261 expire 1584723111 last 1584723034 [159985.446536] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [160040.140647] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbcee32c00, cur 1584723461 expire 1584723311 last 1584723234 [160040.160553] Lustre: Skipped 1 previous similar message [160212.144077] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd937739800, cur 1584723633 expire 1584723483 last 1584723406 [160783.236481] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [160783.245005] Lustre: Skipped 1 previous similar message [160813.284439] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [161010.143064] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934fac00, cur 1584724431 expire 1584724281 last 1584724204 [161010.162980] Lustre: Skipped 1 previous similar message [161013.731896] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [161013.740423] Lustre: Skipped 1 previous similar message [161040.138310] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef5800, cur 1584724461 expire 1584724311 last 1584724234 [161186.083481] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [161240.137517] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9196c00, cur 1584724661 expire 1584724511 last 1584724434 [161240.157433] Lustre: Skipped 1 previous similar message [161412.137560] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf5fee400, cur 1584724833 expire 1584724683 last 1584724606 [161983.809910] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [161983.818444] Lustre: Skipped 1 previous similar message [162013.857807] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [162210.143755] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd829b000, cur 1584725631 expire 1584725481 last 1584725404 [162210.163644] Lustre: Skipped 1 previous similar message [162213.249380] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [162213.257916] Lustre: Skipped 1 previous similar message [162240.150004] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9b9b400, cur 1584725661 expire 1584725511 last 1584725434 [162385.569058] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [162440.132087] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e52d2c00, cur 1584725861 expire 1584725711 last 1584725634 [162440.152000] Lustre: Skipped 1 previous similar message [162612.132061] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbeada5400, cur 1584726033 expire 1584725883 last 1584725806 [162711.478352] LustreError: 49268:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 6d03d9a7-b1f2-4 claims 794624 GRANT, real grant 28672 [162754.783840] LustreError: 49257:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 95038ffe-45ab-4 claims 1441792 GRANT, real grant 0 [163183.231357] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [163183.239906] Lustre: Skipped 1 previous similar message [163213.279395] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [163410.130865] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbeae4d000, cur 1584726831 expire 1584726681 last 1584726604 [163410.150777] Lustre: Skipped 1 previous similar message [163413.694905] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [163413.703437] Lustre: Skipped 1 previous similar message [163440.169221] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbe5b3fc00, cur 1584726861 expire 1584726711 last 1584726634 [163586.078543] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [163640.128061] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb9d1000, cur 1584727061 expire 1584726911 last 1584726834 [163640.147947] Lustre: Skipped 1 previous similar message [163812.138207] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e901b400, cur 1584727233 expire 1584727083 last 1584727006 [164002.543477] LustreError: 49335:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 95eea94a-c8bd-4 claims 16752640 GRANT, real grant 36864 [164383.548698] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [164383.557232] Lustre: Skipped 1 previous similar message [164413.596598] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [164610.128213] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f4800, cur 1584728031 expire 1584727881 last 1584727804 [164610.148120] Lustre: Skipped 1 previous similar message [164613.948127] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [164613.956659] Lustre: Skipped 1 previous similar message [164640.126226] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ebafc400, cur 1584728061 expire 1584727911 last 1584727834 [164785.371704] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [164840.141772] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb91dbb800, cur 1584728261 expire 1584728111 last 1584728034 [164840.161676] Lustre: Skipped 1 previous similar message [165012.129069] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93011ac00, cur 1584728433 expire 1584728283 last 1584728206 [165583.929843] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [165583.938371] Lustre: Skipped 1 previous similar message [165613.977790] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [165810.121297] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938e9a000, cur 1584729231 expire 1584729081 last 1584729004 [165810.141184] Lustre: Skipped 1 previous similar message [165813.305404] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [165813.313932] Lustre: Skipped 1 previous similar message [165840.122580] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938e98000, cur 1584729261 expire 1584729111 last 1584729034 [165985.625117] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [166040.148022] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf3025400, cur 1584729461 expire 1584729311 last 1584729234 [166040.167935] Lustre: Skipped 1 previous similar message [166212.128529] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f576400, cur 1584729633 expire 1584729483 last 1584729406 [166575.091801] LustreError: 49340:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli f3287512-5093-4 claims 8404992 GRANT, real grant 57344 [166715.037519] LustreError: 49251:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 5b050ff2-eda0-4 claims 4169728 GRANT, real grant 1441792 [166783.223382] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [166783.231907] Lustre: Skipped 1 previous similar message [166813.271290] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [167010.130412] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b58400, cur 1584730431 expire 1584730281 last 1584730204 [167010.150317] Lustre: Skipped 1 previous similar message [167013.686861] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [167013.695391] Lustre: Skipped 1 previous similar message [167040.119814] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e0c400, cur 1584730461 expire 1584730311 last 1584730234 [167186.006498] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [167240.120216] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b9e7db000, cur 1584730661 expire 1584730511 last 1584730434 [167240.140121] Lustre: Skipped 1 previous similar message [167412.121441] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fda4437b000, cur 1584730833 expire 1584730683 last 1584730606 [167983.476715] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [167983.485248] Lustre: Skipped 1 previous similar message [168013.524692] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [168180.670252] LustreError: 49314:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 767f4960-85ab-4 claims 16752640 GRANT, real grant 1441792 [168210.118540] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbe7ca1c00, cur 1584731631 expire 1584731481 last 1584731404 [168210.138431] Lustre: Skipped 1 previous similar message [168213.940335] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [168213.948864] Lustre: Skipped 1 previous similar message [168240.144870] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef6c00, cur 1584731661 expire 1584731511 last 1584731434 [168385.363967] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [168424.869240] LustreError: 49388:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dcb1fce2-5124-4 claims 16752640 GRANT, real grant 11210752 [168440.129968] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9610800, cur 1584731861 expire 1584731711 last 1584731634 [168440.149852] Lustre: Skipped 1 previous similar message [168612.117710] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9194400, cur 1584732033 expire 1584731883 last 1584731806 [168799.922233] LustreError: 67482:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 99638e11-6747-4 claims 1093632 GRANT, real grant 172032 [169183.858325] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [169183.866851] Lustre: Skipped 1 previous similar message [169213.906239] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [169410.114441] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5320400, cur 1584732831 expire 1584732681 last 1584732604 [169410.134345] Lustre: Skipped 1 previous similar message [169413.361903] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [169413.370433] Lustre: Skipped 1 previous similar message [169440.115888] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bfa629c00, cur 1584732861 expire 1584732711 last 1584732634 [169585.745464] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [169640.128607] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb78d11c00, cur 1584733061 expire 1584732911 last 1584732834 [169640.148511] Lustre: Skipped 1 previous similar message [169812.127754] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef7c00, cur 1584733233 expire 1584733083 last 1584733006 [170059.567068] LustreError: 102174:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cbf4c94a-83d4-4 claims 4169728 GRANT, real grant 4014080 [170377.374105] LustreError: 49030:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 24485f25-7c65-4 claims 6189056 GRANT, real grant 5038080 [170383.343790] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [170383.352318] Lustre: Skipped 1 previous similar message [170413.391698] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [170610.112796] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eaa87800, cur 1584734031 expire 1584733881 last 1584733804 [170610.132697] Lustre: Skipped 1 previous similar message [170613.807187] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [170613.815713] Lustre: Skipped 1 previous similar message [170640.115264] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd1292800, cur 1584734061 expire 1584733911 last 1584733834 [170786.126808] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [170840.111552] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb9d98fc00, cur 1584734261 expire 1584734111 last 1584734034 [170840.131471] Lustre: Skipped 1 previous similar message [171013.127547] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7219000, cur 1584734434 expire 1584734284 last 1584734207 [171583.789223] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [171583.797776] Lustre: Skipped 1 previous similar message [171613.837105] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [171810.109524] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5ec4400, cur 1584735231 expire 1584735081 last 1584735004 [171810.129440] Lustre: Skipped 1 previous similar message [171813.196704] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [171813.205236] Lustre: Skipped 1 previous similar message [171840.112694] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d5800, cur 1584735261 expire 1584735111 last 1584735034 [171985.484320] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [172040.125447] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd933fcf000, cur 1584735461 expire 1584735311 last 1584735234 [172040.145329] Lustre: Skipped 1 previous similar message [172212.109395] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd3a2a400, cur 1584735633 expire 1584735483 last 1584735406 [172627.659814] Lustre: fir-OST0034: Client 2787f255-32fa-4 (at 10.49.27.33@o2ib1) reconnecting [172627.660170] Lustre: fir-OST003a: Connection restored to 2787f255-32fa-4 (at 10.49.27.33@o2ib1) [172627.660173] Lustre: Skipped 1 previous similar message [172627.682202] Lustre: Skipped 1 previous similar message [172631.425229] LustreError: 49338:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff9ff8262a8050 x1659301860563776/t0(0) o4->8e2770c2-1687-4@10.49.20.4@o2ib1:134/0 lens 488/448 e 1 to 0 dl 1584736074 ref 1 fl Interpret:/0/0 rc 0/0 [172632.081226] LustreError: 49342:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff9ffb772c1050 x1659459914660352/t0(0) o4->222294c0-c625-4@10.49.13.25@o2ib1:145/0 lens 488/448 e 0 to 0 dl 1584736085 ref 1 fl Interpret:/0/0 rc 0/0 [172633.271583] LustreError: 47363:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ff8e52d5800 [172633.282520] Lustre: fir-OST0034: Bulk IO write error with 222294c0-c625-4 (at 10.49.13.25@o2ib1), client will retry: rc = -110 [172633.294019] Lustre: Skipped 4 previous similar messages [172634.378456] LustreError: 47364:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ffbeae49000 [172634.389352] Lustre: fir-OST0030: Bulk IO write error with 8e2770c2-1687-4 (at 10.49.20.4@o2ib1), client will retry: rc = -110 [172635.445217] LustreError: 49295:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk READ req@ffffa00b93c2b050 x1659488371222848/t0(0) o3->f678b43f-5702-4@10.49.27.20@o2ib1:137/0 lens 488/440 e 1 to 0 dl 1584736077 ref 1 fl Interpret:/0/0 rc 0/0 [172635.685045] Lustre: fir-OST0030: Client ab5b24d6-76e3-4 (at 10.49.18.24@o2ib1) reconnecting [172635.693502] Lustre: Skipped 321 previous similar messages [172636.234168] LustreError: 47364:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172636.245048] LustreError: 47366:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172636.263748] LustreError: 47362:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172636.274621] LustreError: 47361:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172637.322136] LustreError: 47366:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172637.333013] LustreError: 47365:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172638.077211] LustreError: 49332:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk READ req@ffff9ffb9483f050 x1659134415571712/t0(0) o3->e8c40140-31a1-4@10.49.28.9@o2ib1:139/0 lens 488/440 e 1 to 0 dl 1584736079 ref 1 fl Interpret:/0/0 rc 0/0 [172638.099440] LustreError: 49332:0:(ldlm_lib.c:3279:target_bulk_io()) Skipped 1 previous similar message [172638.802375] LustreError: 47363:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ffbf29b8000 [172638.813249] LustreError: 47363:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9fe8eb92ec00 [172638.937115] LustreError: 47365:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ff8e52d2c00 [172638.948007] LustreError: 47366:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ffbfe451c00 [172647.626737] LustreError: 47369:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172647.637613] LustreError: 47370:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffffa00bd829ec00 [172647.648832] Lustre: fir-OST0038: Bulk IO read error with f678b43f-5702-4 (at 10.49.27.20@o2ib1), client will retry: rc -110 [172647.660102] Lustre: Skipped 1 previous similar message [172649.230178] LustreError: 47368:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ffbf29b8000 [172649.230181] LustreError: 47369:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9fe8eb92ec00 [172649.230374] Lustre: fir-OST0034: Bulk IO read error with e8c40140-31a1-4 (at 10.49.28.9@o2ib1), client will retry: rc -110 [172649.337860] LustreError: 47370:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ff8e52d2c00 [172649.348740] LustreError: 47370:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ffbfe451c00 [172649.570089] LustreError: 137-5: fir-OST0039_UUID: not available for connect from 10.49.25.23@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [172652.600181] LustreError: 137-5: fir-OST003b_UUID: not available for connect from 10.49.24.33@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [172652.617629] LustreError: Skipped 1 previous similar message [172655.059441] Lustre: fir-OST0036: Client 9d111334-d743-4 (at 10.49.23.27@o2ib1) reconnecting [172655.067883] Lustre: Skipped 291 previous similar messages [172658.179412] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.49.26.26@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [172658.196884] LustreError: Skipped 47 previous similar messages [172666.197533] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.49.22.9@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [172666.214909] LustreError: Skipped 489 previous similar messages [172687.394904] Lustre: fir-OST0032: Client b449688d-0d11-4 (at 10.49.21.21@o2ib1) reconnecting [172687.403353] Lustre: Skipped 108 previous similar messages [172778.154883] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [172778.163426] Lustre: Skipped 940 previous similar messages [173005.109664] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfe456000, cur 1584736426 expire 1584736276 last 1584736199 [173005.129574] Lustre: Skipped 1 previous similar message [173040.105389] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f570c00, cur 1584736461 expire 1584736311 last 1584736234 [173185.866228] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [173185.874755] Lustre: Skipped 3 previous similar messages [173185.916103] LustreError: 102173:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli f46cd69d-9fbd-4 claims 28672 GRANT, real grant 0 [173240.116430] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd933fce400, cur 1584736661 expire 1584736511 last 1584736434 [173240.136316] Lustre: Skipped 1 previous similar message [173412.104533] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb9b3d4c00, cur 1584736833 expire 1584736683 last 1584736606 [173978.536772] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [173978.545303] Lustre: Skipped 1 previous similar message [174205.118532] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5f63c00, cur 1584737626 expire 1584737476 last 1584737399 [174205.138421] Lustre: Skipped 1 previous similar message [174240.103261] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bce6d3400, cur 1584737661 expire 1584737511 last 1584737434 [174440.104429] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf3246800, cur 1584737861 expire 1584737711 last 1584737634 [174440.124344] Lustre: Skipped 1 previous similar message [174612.105903] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93773d400, cur 1584738033 expire 1584737883 last 1584737806 [175179.046771] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [175179.055309] Lustre: Skipped 5 previous similar messages [175405.101616] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad925000, cur 1584738826 expire 1584738676 last 1584738599 [175405.121531] Lustre: Skipped 1 previous similar message [175440.101356] Lustre: fir-OST0036: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad923800, cur 1584738861 expire 1584738711 last 1584738634 [175440.121262] Lustre: Skipped 1 previous similar message [175587.531395] LustreError: 49254:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a67a88ce-e5ae-4 claims 36864 GRANT, real grant 0 [175640.103171] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9195c00, cur 1584739061 expire 1584738911 last 1584738834 [175640.123053] Lustre: Skipped 6 previous similar messages [175693.251852] Lustre: fir-OST0030: Client 15e5b565-193d-4 (at 10.49.30.4@o2ib1) reconnecting [175693.260210] Lustre: Skipped 222 previous similar messages [175703.256257] Lustre: fir-OST0032: Client 7df32ea0-1cd2-4 (at 10.49.24.35@o2ib1) reconnecting [175703.256258] Lustre: fir-OST003a: Client 7df32ea0-1cd2-4 (at 10.49.24.35@o2ib1) reconnecting [175703.256261] Lustre: Skipped 21 previous similar messages [175719.271617] Lustre: fir-OST003a: Client 2ca302e7-d134-4 (at 10.49.22.33@o2ib1) reconnecting [175719.280058] Lustre: Skipped 499 previous similar messages [175728.145566] LustreError: 137-5: fir-OST0033_UUID: not available for connect from 10.49.20.12@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [175728.163021] LustreError: Skipped 504 previous similar messages [175730.992169] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.49.19.1@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [175731.009553] LustreError: Skipped 5 previous similar messages [175735.583393] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.49.30.22@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [175735.600845] LustreError: Skipped 44 previous similar messages [175743.626731] LustreError: 137-5: fir-OST0031_UUID: not available for connect from 10.49.18.20@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [175743.644184] LustreError: Skipped 313 previous similar messages [175749.564027] LustreError: 49266:0:(ldlm_lib.c:3279:target_bulk_io()) @@@ Reconnect on bulk WRITE req@ffff9ffb9483b050 x1660816598548864/t0(0) o4->0dc74af1-84ad-4@10.49.0.61@o2ib1:235/0 lens 488/448 e 1 to 0 dl 1584739195 ref 1 fl Interpret:/0/0 rc 0/0 [175749.586393] LustreError: 49266:0:(ldlm_lib.c:3279:target_bulk_io()) Skipped 2 previous similar messages [175751.165028] Lustre: 48359:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1584739165/real 0] req@ffff9fdbf9ac5580 x1661536257339200/t0(0) o106->fir-OST003a@10.49.8.25@o2ib1:15/16 lens 296/280 e 0 to 1 dl 1584739172 ref 2 fl Rpc:X/0/ffffffff rc 0/-1 [175751.191620] Lustre: 48359:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 20 previous similar messages [175751.755809] Lustre: fir-OST003a: Client cc268c6d-210e-4 (at 10.49.18.21@o2ib1) reconnecting [175751.764250] Lustre: Skipped 685 previous similar messages [175754.441525] LustreError: 47367:0:(events.c:450:server_bulk_callback()) event type 5, status -125, desc ffff9ffbeae4cc00 [175754.452428] Lustre: fir-OST0036: Bulk IO write error with 0dc74af1-84ad-4 (at 10.49.0.61@o2ib1), client will retry: rc = -110 [175759.628202] LustreError: 137-5: fir-OST0033_UUID: not available for connect from 10.49.26.11@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [175759.645659] LustreError: Skipped 771 previous similar messages [175779.098679] Lustre: fir-OST0030: Connection restored to 0f2d3178-05c2-4 (at 10.49.0.64@o2ib1) [175779.107295] Lustre: Skipped 1632 previous similar messages [175792.087475] LustreError: 137-5: fir-OST0033_UUID: not available for connect from 10.49.28.4@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [175792.104878] LustreError: Skipped 409 previous similar messages [175812.112404] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf29b9400, cur 1584739233 expire 1584739083 last 1584739006 [175816.555280] Lustre: fir-OST0030: Client 15e5b565-193d-4 (at 10.49.30.4@o2ib1) reconnecting [175816.563633] Lustre: Skipped 1278 previous similar messages [175970.434691] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 10.49.0.62@o2ib1 ns: filter-fir-OST003a_UUID lock: ffffa005f5d0de80/0xf3793efafb149f94 lrc: 3/0,0 mode: PR/PR res: [0x1180000401:0x717fa3:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 34359738368->18446744073709551615) flags: 0x60000400010020 nid: 10.49.0.62@o2ib1 remote: 0xea77017727c99857 expref: 83524 pid: 49144 timeout: 175970 lvb_type: 1 [176015.101161] Lustre: fir-OST0032: haven't heard from client eb57335a-b614-4 (at 10.49.0.62@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e7d0800, cur 1584739436 expire 1584739286 last 1584739209 [176015.121148] Lustre: Skipped 1 previous similar message [176283.099938] Lustre: fir-OST0038: haven't heard from client 22cd96b1-4e55-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb929c00, cur 1584739704 expire 1584739554 last 1584739477 [176283.119927] Lustre: Skipped 4 previous similar messages [176407.897042] LustreError: 49324:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 23b73773-98ec-4 claims 380928 GRANT, real grant 155648 [176413.700857] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [176413.709382] Lustre: Skipped 882 previous similar messages [176552.061450] LustreError: 49030:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dd1fe72f-e583-4 claims 4730880 GRANT, real grant 364544 [176605.171252] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad925000, cur 1584740026 expire 1584739876 last 1584739799 [176605.191158] Lustre: Skipped 5 previous similar messages [176716.901999] LustreError: 49235:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a09c1f00-b96c-4 claims 4370432 GRANT, real grant 0 [177107.132879] LustreError: 49020:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli dbc3d438-69dd-4 claims 147456 GRANT, real grant 0 [177578.274849] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [177578.283380] Lustre: Skipped 4 previous similar messages [177656.096553] Lustre: fir-OST0036: haven't heard from client 4119ac7e-3e26-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d2c00, cur 1584741077 expire 1584740927 last 1584740850 [177656.116549] Lustre: Skipped 5 previous similar messages [177962.114952] LustreError: 49139:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli b73e1dab-2ad6-4 claims 1687552 GRANT, real grant 483328 [178300.099131] LustreError: 49206:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli cc51ef17-5ec1-4 claims 16752640 GRANT, real grant 0 [178779.041009] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [178779.049539] Lustre: Skipped 11 previous similar messages [178919.668394] LustreError: 102173:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli b222a878-96be-4 claims 16723968 GRANT, real grant 114688 [179005.096156] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a1400, cur 1584742426 expire 1584742276 last 1584742199 [179005.116070] Lustre: Skipped 11 previous similar messages [179123.142169] LustreError: 49237:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli d0ed2bdc-7329-4 claims 28672 GRANT, real grant 0 [179240.106152] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf77b4800, cur 1584742661 expire 1584742511 last 1584742434 [179240.126034] Lustre: Skipped 2 previous similar messages [179412.093489] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef15c00, cur 1584742833 expire 1584742683 last 1584742606 [179494.989900] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [179494.998519] Lustre: Skipped 10 previous similar messages [179582.975379] LustreError: 49150:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3da00d75-6c3b-4 claims 106496 GRANT, real grant 0 [180205.123523] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00befa7e800, cur 1584743626 expire 1584743476 last 1584743399 [180205.143426] Lustre: Skipped 7 previous similar messages [180213.886719] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [180213.895348] Lustre: Skipped 3 previous similar messages [180278.044215] LustreError: 49249:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli c54d471d-70f7-4 claims 4141056 GRANT, real grant 126976 [181178.909058] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [181178.917606] Lustre: Skipped 2 previous similar messages [181405.089640] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbeae4dc00, cur 1584744826 expire 1584744676 last 1584744599 [181405.109526] Lustre: Skipped 5 previous similar messages [181737.747015] LustreError: 49257:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3e4eec45-c061-4 claims 1253376 GRANT, real grant 221184 [181793.565948] LustreError: 49373:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 86db2c50-4e1f-4 claims 86016 GRANT, real grant 45056 [181910.750079] LustreError: 49296:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6dd0a0db-fe9d-4 claims 32768 GRANT, real grant 28672 [182378.267143] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [182378.275676] Lustre: Skipped 17 previous similar messages [182605.086569] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c5c00, cur 1584746026 expire 1584745876 last 1584745799 [182605.106480] Lustre: Skipped 11 previous similar messages [183098.125690] LustreError: 49279:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 9891086f-7733-4 claims 4644864 GRANT, real grant 3321856 [183578.520940] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [183578.529469] Lustre: Skipped 11 previous similar messages [183805.106242] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9611c00, cur 1584747226 expire 1584747076 last 1584746999 [183805.126127] Lustre: Skipped 11 previous similar messages [184779.030460] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [184779.038993] Lustre: Skipped 5 previous similar messages [185005.089765] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a6c00, cur 1584748426 expire 1584748276 last 1584748199 [185005.109692] Lustre: Skipped 5 previous similar messages [185978.515792] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [185978.524335] Lustre: Skipped 5 previous similar messages [186205.079719] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf72b3000, cur 1584749626 expire 1584749476 last 1584749399 [186205.099603] Lustre: Skipped 5 previous similar messages [187179.025311] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [187179.033842] Lustre: Skipped 5 previous similar messages [187405.086450] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb92f000, cur 1584750826 expire 1584750676 last 1584750599 [187405.106368] Lustre: Skipped 5 previous similar messages [187794.214325] LustreError: 48964:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a61ba52f-0e9d-4 claims 253952 GRANT, real grant 32768 [188378.511382] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [188378.519912] Lustre: Skipped 5 previous similar messages [188605.090676] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf57f6800, cur 1584752026 expire 1584751876 last 1584751799 [188605.110560] Lustre: Skipped 5 previous similar messages [189474.246402] LustreError: 49133:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 56bdeeef-d646-4 claims 28672 GRANT, real grant 0 [189561.409365] LustreError: 49400:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli dff180c1-de42-4 claims 155648 GRANT, real grant 0 [189579.149531] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [189579.158068] Lustre: Skipped 10 previous similar messages [189806.086477] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5977000, cur 1584753227 expire 1584753077 last 1584753000 [189806.106409] Lustre: Skipped 11 previous similar messages [190073.557263] LustreError: 49014:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 110592 GRANT, real grant 28672 [190229.220832] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [190229.229449] Lustre: Skipped 10 previous similar messages [191006.070308] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f576000, cur 1584754427 expire 1584754277 last 1584754200 [191006.090296] Lustre: Skipped 11 previous similar messages [191014.059554] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [191014.068081] Lustre: Skipped 3 previous similar messages [191842.323726] LustreError: 49312:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli dbc3d438-69dd-4 claims 229376 GRANT, real grant 147456 [191980.106219] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [191980.114774] Lustre: Skipped 8 previous similar messages [192207.070797] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5f62800, cur 1584755628 expire 1584755478 last 1584755401 [192207.090683] Lustre: Skipped 11 previous similar messages [192473.930371] LustreError: 49312:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 2c8e6de5-28f9-4 claims 5619712 GRANT, real grant 0 [193180.489754] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [193180.498320] Lustre: Skipped 5 previous similar messages [193407.080515] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf0ee8c00, cur 1584756828 expire 1584756678 last 1584756601 [193407.100424] Lustre: Skipped 5 previous similar messages [193490.334808] LustreError: 49134:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a61ba52f-0e9d-4 claims 311296 GRANT, real grant 253952 [193520.137742] LustreError: 49284:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli be752b6f-e9b6-4 claims 229376 GRANT, real grant 94208 [193683.083624] LustreError: 49255:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 6e05ab9f-8c33-4 claims 970752 GRANT, real grant 77824 [194380.871638] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [194380.880164] Lustre: Skipped 5 previous similar messages [194607.064767] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf91afc00, cur 1584758028 expire 1584757878 last 1584757801 [194607.084668] Lustre: Skipped 5 previous similar messages [194777.612249] LustreError: 49264:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli dbc3d438-69dd-4 claims 1482752 GRANT, real grant 229376 [195320.466071] LustreError: 48331:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3df0b565-ed22-4 claims 32768 GRANT, real grant 0 [195580.230239] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [195580.238769] Lustre: Skipped 5 previous similar messages [195807.067578] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf29bc000, cur 1584759228 expire 1584759078 last 1584759001 [195807.087484] Lustre: Skipped 5 previous similar messages [196780.612127] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [196780.620658] Lustre: Skipped 5 previous similar messages [197007.061775] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00a38a54c00, cur 1584760428 expire 1584760278 last 1584760201 [197007.081673] Lustre: Skipped 5 previous similar messages [197981.057826] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [197981.066353] Lustre: Skipped 11 previous similar messages [198207.060496] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd939e76800, cur 1584761628 expire 1584761478 last 1584761401 [198207.080445] Lustre: Skipped 11 previous similar messages [199116.062619] Lustre: fir-OST0030: haven't heard from client 83ad7733-be39-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9b9e800, cur 1584762537 expire 1584762387 last 1584762310 [199116.082588] Lustre: Skipped 5 previous similar messages [199180.607605] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [199180.616141] Lustre: Skipped 5 previous similar messages [199622.996688] LustreError: 49135:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a61ba52f-0e9d-4 claims 671744 GRANT, real grant 311296 [199813.067504] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e0d800, cur 1584763234 expire 1584763084 last 1584763007 [199813.087413] Lustre: Skipped 9 previous similar messages [200033.088227] LustreError: 49373:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a67a88ce-e5ae-4 claims 319488 GRANT, real grant 36864 [200166.533978] LustreError: 49232:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli fab83063-bce6-4 claims 16752640 GRANT, real grant 151552 [200381.117624] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [200381.126153] Lustre: Skipped 11 previous similar messages [200608.077672] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d0cb800, cur 1584764029 expire 1584763879 last 1584763802 [200608.097581] Lustre: Skipped 1 previous similar message [201581.563640] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [201581.572177] Lustre: Skipped 5 previous similar messages [201808.053503] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5320800, cur 1584765229 expire 1584765079 last 1584765002 [201808.073399] Lustre: Skipped 5 previous similar messages [202292.388730] LustreError: 49310:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a09c1f00-b96c-4 claims 16752640 GRANT, real grant 4370432 [202651.935357] LustreError: 49134:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli b50e5f2f-f7a2-4 claims 28672 GRANT, real grant 0 [202781.945570] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [202781.954100] Lustre: Skipped 5 previous similar messages [203008.065351] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5323c00, cur 1584766429 expire 1584766279 last 1584766202 [203008.085253] Lustre: Skipped 5 previous similar messages [203757.651290] LustreError: 49306:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 87383dee-e650-4 claims 28672 GRANT, real grant 0 [203981.431129] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [203981.439682] Lustre: Skipped 5 previous similar messages [204208.048623] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbed0b5800, cur 1584767629 expire 1584767479 last 1584767402 [204208.068518] Lustre: Skipped 5 previous similar messages [204650.251184] LustreError: 49400:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli c3fa2b6b-5cac-4 claims 16752640 GRANT, real grant 0 [205181.812693] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [205181.821231] Lustre: Skipped 5 previous similar messages [205408.046507] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e901a000, cur 1584768829 expire 1584768679 last 1584768602 [205408.066419] Lustre: Skipped 5 previous similar messages [206141.633215] LustreError: 102166:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 86db2c50-4e1f-4 claims 98304 GRANT, real grant 86016 [206381.170344] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [206381.178872] Lustre: Skipped 5 previous similar messages [206501.043855] Lustre: fir-OST0038: haven't heard from client e312085b-9063-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd932b2a800, cur 1584769922 expire 1584769772 last 1584769695 [206501.063838] Lustre: Skipped 5 previous similar messages [206618.061640] LustreError: 49206:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 24485f25-7c65-4 claims 16752640 GRANT, real grant 6189056 [207581.296702] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [207581.305266] Lustre: Skipped 5 previous similar messages [207808.046310] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a6c00, cur 1584771229 expire 1584771079 last 1584771002 [207808.066188] Lustre: Skipped 11 previous similar messages [207878.621622] LustreError: 67601:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli eac7d7e6-7a39-4 claims 10715136 GRANT, real grant 8413184 [208040.065406] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bee9ca800, cur 1584771461 expire 1584771311 last 1584771234 [208040.085312] Lustre: Skipped 2 previous similar messages [208213.047788] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf72b3400, cur 1584771634 expire 1584771484 last 1584771407 [208781.551123] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [208781.559658] Lustre: Skipped 11 previous similar messages [209008.040275] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ebaaf400, cur 1584772429 expire 1584772279 last 1584772202 [209008.060177] Lustre: Skipped 1 previous similar message [209348.963845] LustreError: 102175:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 99638e11-6747-4 claims 4169728 GRANT, real grant 1093632 [209734.521588] LustreError: 49265:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 5616204e-fc92-4 claims 5431296 GRANT, real grant 880640 [209981.804589] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [209981.813170] Lustre: Skipped 5 previous similar messages [210061.418200] LustreError: 49190:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 86db2c50-4e1f-4 claims 110592 GRANT, real grant 98304 [210208.037077] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb79266800, cur 1584773629 expire 1584773479 last 1584773402 [210208.056957] Lustre: Skipped 5 previous similar messages [211182.070774] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [211182.079310] Lustre: Skipped 5 previous similar messages [211409.036123] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb78d12000, cur 1584774830 expire 1584774680 last 1584774603 [211409.056004] Lustre: Skipped 5 previous similar messages [211929.112118] LustreError: 49299:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 7940c6f6-66d7-4 claims 28672 GRANT, real grant 0 [212381.160479] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [212381.169005] Lustre: Skipped 5 previous similar messages [212608.055325] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d4800, cur 1584776029 expire 1584775879 last 1584775802 [212608.075231] Lustre: Skipped 5 previous similar messages [212812.147514] LustreError: 91448:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 2db0db09-6b3c-4 claims 65536 GRANT, real grant 0 [213413.234406] LustreError: 91449:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 87383dee-e650-4 claims 225280 GRANT, real grant 28672 [213581.350982] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [213581.359508] Lustre: Skipped 5 previous similar messages [213808.042727] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938548800, cur 1584777229 expire 1584777079 last 1584777002 [213808.062617] Lustre: Skipped 5 previous similar messages [213876.873825] LustreError: 49216:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 6e05ab9f-8c33-4 claims 5521408 GRANT, real grant 970752 [214199.568514] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [214199.577131] Lustre: Skipped 10 previous similar messages [214813.540323] Lustre: fir-OST0036: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [214813.548863] Lustre: Skipped 1 previous similar message [215008.031275] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93011a800, cur 1584778429 expire 1584778279 last 1584778202 [215008.051158] Lustre: Skipped 11 previous similar messages [215981.730725] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [215981.739259] Lustre: Skipped 4 previous similar messages [216208.032410] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef11000, cur 1584779629 expire 1584779479 last 1584779402 [216208.052341] Lustre: Skipped 5 previous similar messages [217181.089459] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [217181.098047] Lustre: Skipped 5 previous similar messages [217408.034865] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b86605000, cur 1584780829 expire 1584780679 last 1584780602 [217408.054752] Lustre: Skipped 5 previous similar messages [217955.184017] LustreError: 49365:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3a5b08eb-e519-4 claims 16728064 GRANT, real grant 532480 [218114.673979] LustreError: 67601:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli bc3a5e42-5951-4 claims 9310208 GRANT, real grant 6135808 [218381.599209] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [218381.607746] Lustre: Skipped 5 previous similar messages [218608.027482] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd1f1503400, cur 1584782029 expire 1584781879 last 1584781802 [218608.047384] Lustre: Skipped 5 previous similar messages [219581.980955] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [219581.989512] Lustre: Skipped 5 previous similar messages [219768.155910] LustreError: 49243:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 75daa1e6-b38d-4 claims 28672 GRANT, real grant 0 [219808.020141] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5e74c00, cur 1584783229 expire 1584783079 last 1584783002 [219808.040054] Lustre: Skipped 5 previous similar messages [220781.339737] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [220781.348262] Lustre: Skipped 5 previous similar messages [221008.018981] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5ac800, cur 1584784429 expire 1584784279 last 1584784202 [221008.038904] Lustre: Skipped 5 previous similar messages [221981.594529] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [221981.603061] Lustre: Skipped 5 previous similar messages [222208.021065] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf7fe8000, cur 1584785629 expire 1584785479 last 1584785402 [222208.041012] Lustre: Skipped 5 previous similar messages [223181.975507] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [223181.984040] Lustre: Skipped 5 previous similar messages [223408.014442] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad924c00, cur 1584786829 expire 1584786679 last 1584786602 [223408.034342] Lustre: Skipped 5 previous similar messages [224381.461626] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [224381.470156] Lustre: Skipped 5 previous similar messages [224608.018876] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80294c00, cur 1584788029 expire 1584787879 last 1584787802 [224608.038770] Lustre: Skipped 5 previous similar messages [225577.072134] LustreError: 49206:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 2db0db09-6b3c-4 claims 16752640 GRANT, real grant 442368 [225581.716318] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [225581.724851] Lustre: Skipped 5 previous similar messages [225808.013485] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf57f7000, cur 1584789229 expire 1584789079 last 1584789002 [225808.033433] Lustre: Skipped 5 previous similar messages [226272.762417] LustreError: 48896:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 4a774151-5198-4 claims 4128768 GRANT, real grant 155648 [226491.444388] LustreError: 49373:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 240ede69-17ef-4 claims 9138176 GRANT, real grant 4218880 [226782.098142] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [226782.106683] Lustre: Skipped 5 previous similar messages [227009.009449] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9366cc000, cur 1584790430 expire 1584790280 last 1584790203 [227009.029358] Lustre: Skipped 5 previous similar messages [227706.162312] LustreError: 67483:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 6e05ab9f-8c33-4 claims 7741440 GRANT, real grant 5521408 [227982.543768] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [227982.552296] Lustre: Skipped 5 previous similar messages [228209.016070] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934f9c00, cur 1584791630 expire 1584791480 last 1584791403 [228209.035974] Lustre: Skipped 5 previous similar messages [229182.799150] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [229182.807719] Lustre: Skipped 5 previous similar messages [229409.006068] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf902f800, cur 1584792830 expire 1584792680 last 1584792603 [229409.026055] Lustre: Skipped 5 previous similar messages [230383.052534] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [230383.061063] Lustre: Skipped 5 previous similar messages [230610.046369] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b5d000, cur 1584794031 expire 1584793881 last 1584793804 [230610.066275] Lustre: Skipped 5 previous similar messages [231153.750284] Lustre: fir-OST0030: Connection restored to e71e196d-00dc-4 (at 10.50.5.33@o2ib2) [231153.758895] Lustre: Skipped 10 previous similar messages [231809.658707] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [231809.667235] Lustre: Skipped 3 previous similar messages [231810.002838] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5322800, cur 1584795231 expire 1584795081 last 1584795004 [231810.022747] Lustre: Skipped 11 previous similar messages [232252.060009] LustreError: 99543:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 75daa1e6-b38d-4 claims 57344 GRANT, real grant 0 [232783.434220] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [232783.442747] Lustre: Skipped 2 previous similar messages [233010.011770] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bfa62ac00, cur 1584796431 expire 1584796281 last 1584796204 [233010.031743] Lustre: Skipped 5 previous similar messages [233983.943871] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [233983.952405] Lustre: Skipped 5 previous similar messages [234210.006658] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbec5f9800, cur 1584797631 expire 1584797481 last 1584797404 [234210.026563] Lustre: Skipped 5 previous similar messages [235183.428505] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [235183.437038] Lustre: Skipped 5 previous similar messages [235409.995569] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9386d0400, cur 1584798831 expire 1584798681 last 1584798604 [235410.015476] Lustre: Skipped 5 previous similar messages [236383.682172] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [236383.690703] Lustre: Skipped 5 previous similar messages [236610.003544] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9512800, cur 1584800031 expire 1584799881 last 1584799804 [236610.023449] Lustre: Skipped 5 previous similar messages [237583.052916] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [237583.061442] Lustre: Skipped 5 previous similar messages [237809.990075] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931328c00, cur 1584801231 expire 1584801081 last 1584801004 [237810.009984] Lustre: Skipped 5 previous similar messages [238783.422621] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [238783.431153] Lustre: Skipped 5 previous similar messages [239009.991525] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ebaacc00, cur 1584802431 expire 1584802281 last 1584802204 [239010.011436] Lustre: Skipped 5 previous similar messages [239983.806108] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [239983.814643] Lustre: Skipped 5 previous similar messages [240209.986421] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad926c00, cur 1584803631 expire 1584803481 last 1584803404 [240210.006358] Lustre: Skipped 5 previous similar messages [240999.326802] LustreError: 49254:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 67876c18-c2e3-4 claims 122880 GRANT, real grant 90112 [241043.985805] Lustre: fir-OST0034: haven't heard from client 55eb8d67-6a00-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93132c000, cur 1584804465 expire 1584804315 last 1584804238 [241044.005818] Lustre: Skipped 5 previous similar messages [241104.063571] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [241104.063572] Lustre: fir-OST0034: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [241104.063574] Lustre: Skipped 5 previous similar messages [241104.086205] Lustre: Skipped 4 previous similar messages [241812.984076] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5321c00, cur 1584805234 expire 1584805084 last 1584805007 [241813.003990] Lustre: Skipped 9 previous similar messages [242383.097895] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [242383.106428] Lustre: Skipped 6 previous similar messages [242609.992953] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93132ac00, cur 1584806031 expire 1584805881 last 1584805804 [242610.012857] Lustre: Skipped 1 previous similar message [242611.641476] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [242611.650007] Lustre: Skipped 2 previous similar messages [242786.138125] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [243583.928757] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [243583.937325] Lustre: Skipped 1 previous similar message [243810.000423] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c4800, cur 1584807231 expire 1584807081 last 1584807004 [243810.020325] Lustre: Skipped 5 previous similar messages [244783.542515] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [244783.551051] Lustre: Skipped 5 previous similar messages [245009.989206] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd932b2b400, cur 1584808431 expire 1584808281 last 1584808204 [245010.009139] Lustre: Skipped 5 previous similar messages [245318.887017] LustreError: 48891:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 240ede69-17ef-4 claims 16752640 GRANT, real grant 9138176 [245476.376054] LustreError: 49292:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 56bdeeef-d646-4 claims 303104 GRANT, real grant 28672 [245983.796102] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [245983.804630] Lustre: Skipped 5 previous similar messages [246209.980498] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e0b400, cur 1584809631 expire 1584809481 last 1584809404 [246210.000406] Lustre: Skipped 5 previous similar messages [246916.504662] LustreError: 49358:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli f7fe261e-a413-4 claims 28672 GRANT, real grant 0 [247184.049485] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [247184.058049] Lustre: Skipped 5 previous similar messages [247410.976312] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c1800, cur 1584810832 expire 1584810682 last 1584810605 [247410.996215] Lustre: Skipped 5 previous similar messages [247505.326573] LustreError: 49298:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 75daa1e6-b38d-4 claims 221184 GRANT, real grant 57344 [247673.165509] LustreError: 49075:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 75daa1e6-b38d-4 claims 57344 GRANT, real grant 28672 [248317.681534] LustreError: 48944:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli c2ca4c5a-e67e-4 claims 28672 GRANT, real grant 0 [248384.302018] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [248384.310558] Lustre: Skipped 5 previous similar messages [248610.974349] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf91ae400, cur 1584812032 expire 1584811882 last 1584811805 [248610.994261] Lustre: Skipped 5 previous similar messages [249584.427251] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [249584.435790] Lustre: Skipped 5 previous similar messages [249810.979493] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938eaf000, cur 1584813232 expire 1584813082 last 1584813005 [249810.999429] Lustre: Skipped 5 previous similar messages [250784.808407] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [250784.816940] Lustre: Skipped 5 previous similar messages [250891.206635] LustreError: 51031:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 5cfc0423-37f2-4 claims 7118848 GRANT, real grant 1835008 [251010.986912] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9374a0000, cur 1584814432 expire 1584814282 last 1584814205 [251011.006803] Lustre: Skipped 5 previous similar messages [251984.294790] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [251984.303338] Lustre: Skipped 5 previous similar messages [252005.559148] LustreError: 49273:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6dd0a0db-fe9d-4 claims 970752 GRANT, real grant 32768 [252169.263225] LustreError: 49325:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 1548288 GRANT, real grant 598016 [252206.758103] LustreError: 49298:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 2c8e6de5-28f9-4 claims 16752640 GRANT, real grant 5619712 [252210.958863] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938e98400, cur 1584815632 expire 1584815482 last 1584815405 [252210.978752] Lustre: Skipped 5 previous similar messages [253184.674991] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [253184.683586] Lustre: Skipped 5 previous similar messages [253410.961645] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf9f09400, cur 1584816832 expire 1584816682 last 1584816605 [253410.981557] Lustre: Skipped 5 previous similar messages [254384.287650] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [254384.296181] Lustre: Skipped 5 previous similar messages [254610.960759] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd933fcc400, cur 1584818032 expire 1584817882 last 1584817805 [254610.980646] Lustre: Skipped 5 previous similar messages [254983.865587] LustreError: 48900:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 75daa1e6-b38d-4 claims 303104 GRANT, real grant 57344 [255584.796499] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [255584.805026] Lustre: Skipped 5 previous similar messages [255810.956911] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c2000, cur 1584819232 expire 1584819082 last 1584819005 [255810.976813] Lustre: Skipped 5 previous similar messages [256784.154249] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [256784.162779] Lustre: Skipped 5 previous similar messages [257010.960542] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5f60400, cur 1584820432 expire 1584820282 last 1584820205 [257010.980455] Lustre: Skipped 5 previous similar messages [257984.793533] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [257984.802066] Lustre: Skipped 5 previous similar messages [258210.960615] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd939e72800, cur 1584821632 expire 1584821482 last 1584821405 [258210.980514] Lustre: Skipped 5 previous similar messages [259185.047767] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [259185.056296] Lustre: Skipped 5 previous similar messages [259411.955260] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a6400, cur 1584822833 expire 1584822683 last 1584822606 [259411.975193] Lustre: Skipped 5 previous similar messages [260385.494003] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [260385.502529] Lustre: Skipped 5 previous similar messages [260611.940761] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef11400, cur 1584824033 expire 1584823883 last 1584823806 [260611.960648] Lustre: Skipped 5 previous similar messages [261152.899673] LustreError: 48900:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 275b0d00-9e41-4 claims 155648 GRANT, real grant 28672 [261524.529299] LustreError: 49373:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli c2ca4c5a-e67e-4 claims 1441792 GRANT, real grant 0 [261585.875447] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [261585.883978] Lustre: Skipped 5 previous similar messages [261811.935800] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef5c00, cur 1584825233 expire 1584825083 last 1584825006 [261811.955715] Lustre: Skipped 5 previous similar messages [262785.234270] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [262785.242806] Lustre: Skipped 5 previous similar messages [263011.933862] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5ec4400, cur 1584826433 expire 1584826283 last 1584826206 [263011.953745] Lustre: Skipped 5 previous similar messages [263985.687025] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [263985.695558] Lustre: Skipped 5 previous similar messages [264211.931962] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9611c00, cur 1584827633 expire 1584827483 last 1584827406 [264211.951874] Lustre: Skipped 5 previous similar messages [264353.916283] LustreError: 49296:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli e19e4c6e-1211-4 claims 2060288 GRANT, real grant 397312 [265185.997327] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [265186.005856] Lustre: Skipped 5 previous similar messages [265405.383662] LustreError: 49306:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli f44d5096-53d1-4 claims 536576 GRANT, real grant 28672 [265412.928814] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7a3bcc00, cur 1584828834 expire 1584828684 last 1584828607 [265412.948696] Lustre: Skipped 5 previous similar messages [266044.725970] LustreError: 49307:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2f954eed-4b6c-4 claims 32768 GRANT, real grant 0 [266305.952673] LustreError: 49265:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli be752b6f-e9b6-4 claims 311296 GRANT, real grant 204800 [266386.443445] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [266386.451970] Lustre: Skipped 5 previous similar messages [266612.928265] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931935400, cur 1584830034 expire 1584829884 last 1584829807 [266612.948175] Lustre: Skipped 5 previous similar messages [266946.608206] LustreError: 49235:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 88de242f-0ae5-4 claims 16678912 GRANT, real grant 4096 [267586.826696] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [267586.835229] Lustre: Skipped 5 previous similar messages [267813.015536] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbba7a5800, cur 1584831234 expire 1584831084 last 1584831007 [267813.035439] Lustre: Skipped 5 previous similar messages [267968.633672] LustreError: 49312:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 72866633-325f-4 claims 151552 GRANT, real grant 0 [268786.184900] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [268786.193436] Lustre: Skipped 5 previous similar messages [269012.929983] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef2000, cur 1584832434 expire 1584832284 last 1584832207 [269012.949891] Lustre: Skipped 5 previous similar messages [269426.408358] LustreError: 49256:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli dbc3d438-69dd-4 claims 15974400 GRANT, real grant 704512 [269520.420365] LustreError: 49291:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli c2ca4c5a-e67e-4 claims 155648 GRANT, real grant 28672 [269986.567199] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [269986.575733] Lustre: Skipped 5 previous similar messages [270212.922971] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eaa81000, cur 1584833634 expire 1584833484 last 1584833407 [270212.942877] Lustre: Skipped 5 previous similar messages [270797.011692] LustreError: 49312:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli e19e4c6e-1211-4 claims 10813440 GRANT, real grant 1937408 [271186.948329] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [271186.956865] Lustre: Skipped 5 previous similar messages [271413.924061] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb6f11b000, cur 1584834835 expire 1584834685 last 1584834608 [271413.943968] Lustre: Skipped 5 previous similar messages [272380.344125] LustreError: 49357:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli f46cd69d-9fbd-4 claims 10346496 GRANT, real grant 28672 [272386.179699] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [272386.188234] Lustre: Skipped 5 previous similar messages [272612.914668] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938548c00, cur 1584836034 expire 1584835884 last 1584835807 [272612.934563] Lustre: Skipped 5 previous similar messages [273586.434334] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [273586.442864] Lustre: Skipped 5 previous similar messages [273812.912459] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd1291400, cur 1584837234 expire 1584837084 last 1584837007 [273812.932345] Lustre: Skipped 5 previous similar messages [274786.561286] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [274786.569866] Lustre: Skipped 5 previous similar messages [274819.356682] LustreError: 49281:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 17f2724d-4d81-4 claims 45056 GRANT, real grant 0 [275012.911332] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9617400, cur 1584838434 expire 1584838284 last 1584838207 [275012.931217] Lustre: Skipped 5 previous similar messages [275098.646142] LustreError: 49286:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli f5a2750a-4518-4 claims 28672 GRANT, real grant 0 [275899.909638] Lustre: fir-OST0032: haven't heard from client a47ccd42-c337-4 (at 10.50.17.43@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7dcbb800, cur 1584839321 expire 1584839171 last 1584839094 [275899.929695] Lustre: Skipped 5 previous similar messages [275923.125082] Lustre: fir-OST0030: Connection restored to a47ccd42-c337-4 (at 10.50.17.43@o2ib2) [275923.133792] Lustre: Skipped 10 previous similar messages [276296.442891] LustreError: 50849:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 12242944 GRANT, real grant 10031104 [276612.908399] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5aa400, cur 1584840034 expire 1584839884 last 1584839807 [276612.928310] Lustre: Skipped 9 previous similar messages [277186.443888] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [277186.452417] Lustre: Skipped 6 previous similar messages [277412.765621] Lustre: fir-OST0034: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [277412.774149] Lustre: Skipped 2 previous similar messages [277412.905885] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9b9a000, cur 1584840834 expire 1584840684 last 1584840607 [277412.925774] Lustre: Skipped 1 previous similar message [277586.140448] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [278386.877550] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [278386.886078] Lustre: Skipped 1 previous similar message [278612.905554] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf57f0000, cur 1584842034 expire 1584841884 last 1584841807 [278612.925460] Lustre: Skipped 5 previous similar messages [279586.298504] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [279586.307041] Lustre: Skipped 11 previous similar messages [279812.911779] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd3b66800, cur 1584843234 expire 1584843084 last 1584843007 [279812.931689] Lustre: Skipped 11 previous similar messages [280113.018893] LustreError: 48327:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 6e05ab9f-8c33-4 claims 2129920 GRANT, real grant 81920 [280623.717124] LustreError: 49204:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 95038ffe-45ab-4 claims 16752640 GRANT, real grant 1441792 [280786.681335] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [280786.689860] Lustre: Skipped 5 previous similar messages [281012.900546] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbec5ffc00, cur 1584844434 expire 1584844284 last 1584844207 [281012.920480] Lustre: Skipped 5 previous similar messages [281986.168225] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [281986.176750] Lustre: Skipped 5 previous similar messages [282212.916151] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b86601800, cur 1584845634 expire 1584845484 last 1584845407 [282212.936057] Lustre: Skipped 5 previous similar messages [283186.678408] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [283186.686936] Lustre: Skipped 5 previous similar messages [283412.902900] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93132e400, cur 1584846834 expire 1584846684 last 1584846607 [283412.922795] Lustre: Skipped 5 previous similar messages [284386.164209] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [284386.172750] Lustre: Skipped 5 previous similar messages [284580.638986] LustreError: 67478:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 7940c6f6-66d7-4 claims 1441792 GRANT, real grant 28672 [284612.896230] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e52d3800, cur 1584848034 expire 1584847884 last 1584847807 [284612.916132] Lustre: Skipped 5 previous similar messages [285586.559987] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [285586.568519] Lustre: Skipped 5 previous similar messages [285812.913226] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938e9c400, cur 1584849234 expire 1584849084 last 1584849007 [285812.933117] Lustre: Skipped 5 previous similar messages [286767.280009] LustreError: 48891:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 385024 GRANT, real grant 110592 [286786.031985] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [286786.040517] Lustre: Skipped 5 previous similar messages [287012.890253] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c7400, cur 1584850434 expire 1584850284 last 1584850207 [287012.910163] Lustre: Skipped 5 previous similar messages [287986.286065] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [287986.294599] Lustre: Skipped 5 previous similar messages [288212.908175] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf721bc00, cur 1584851634 expire 1584851484 last 1584851407 [288212.928061] Lustre: Skipped 5 previous similar messages [289186.733998] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [289186.742529] Lustre: Skipped 5 previous similar messages [289412.896535] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9445800, cur 1584852834 expire 1584852684 last 1584852607 [289412.916432] Lustre: Skipped 5 previous similar messages [290386.026551] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [290386.035082] Lustre: Skipped 5 previous similar messages [290612.885721] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbba7a0c00, cur 1584854034 expire 1584853884 last 1584853807 [290612.905628] Lustre: Skipped 5 previous similar messages [290647.017002] LustreError: 49310:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a67a88ce-e5ae-4 claims 11575296 GRANT, real grant 319488 [290856.350762] LustreError: 49015:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli daeb2854-fef7-4 claims 16752640 GRANT, real grant 28672 [291437.712620] LustreError: 49135:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6dd0a0db-fe9d-4 claims 2973696 GRANT, real grant 970752 [291586.407716] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [291586.416244] Lustre: Skipped 5 previous similar messages [291789.354742] LustreError: 67483:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 35cd2c2f-e02c-4 claims 10579968 GRANT, real grant 28672 [291812.895544] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbd3b64400, cur 1584855234 expire 1584855084 last 1584855007 [291812.915424] Lustre: Skipped 5 previous similar messages [291890.495241] LustreError: 49281:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 17f2724d-4d81-4 claims 10387456 GRANT, real grant 45056 [292786.789607] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [292786.798135] Lustre: Skipped 5 previous similar messages [293012.882478] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9366ca800, cur 1584856434 expire 1584856284 last 1584856207 [293012.902368] Lustre: Skipped 5 previous similar messages [293421.893761] LustreError: 49340:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 741376 GRANT, real grant 385024 [293986.916316] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [293986.924845] Lustre: Skipped 5 previous similar messages [294213.875465] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffca9b41c00, cur 1584857635 expire 1584857485 last 1584857408 [294213.895360] Lustre: Skipped 5 previous similar messages [295186.274778] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [295186.283307] Lustre: Skipped 5 previous similar messages [295412.888801] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd935e70400, cur 1584858834 expire 1584858684 last 1584858607 [295412.908715] Lustre: Skipped 5 previous similar messages [295710.735949] LustreError: 49273:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 87383dee-e650-4 claims 16728064 GRANT, real grant 200704 [295743.983593] LustreError: 49228:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 95038ffe-45ab-4 claims 16752640 GRANT, real grant 40960 [296386.399607] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [296386.408143] Lustre: Skipped 5 previous similar messages [296612.878196] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf1f0a000, cur 1584860034 expire 1584859884 last 1584859807 [296612.898182] Lustre: Skipped 5 previous similar messages [297057.100227] LustreError: 49220:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 00d873ba-49ea-4 claims 303104 GRANT, real grant 28672 [297168.459776] LustreError: 49133:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 00d873ba-49ea-4 claims 32768 GRANT, real grant 28672 [297168.472294] LustreError: 49133:0:(tgt_grant.c:758:tgt_grant_check()) Skipped 1 previous similar message [297586.577490] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [297586.586019] Lustre: Skipped 5 previous similar messages [297812.880147] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b59400, cur 1584861234 expire 1584861084 last 1584861007 [297812.900053] Lustre: Skipped 5 previous similar messages [298786.907965] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [298786.916505] Lustre: Skipped 5 previous similar messages [299013.866625] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934f9400, cur 1584862435 expire 1584862285 last 1584862208 [299013.886536] Lustre: Skipped 5 previous similar messages [299609.598137] LustreError: 49313:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli f7fe261e-a413-4 claims 1785856 GRANT, real grant 28672 [299986.266288] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [299986.274826] Lustre: Skipped 5 previous similar messages [300212.866914] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b5f800, cur 1584863634 expire 1584863484 last 1584863407 [300212.886831] Lustre: Skipped 5 previous similar messages [300581.131954] md: data-check of RAID array md6 [300587.110292] md: data-check of RAID array md2 [300593.183683] md: data-check of RAID array md10 [300599.330714] md: data-check of RAID array md4 [300605.348342] md: data-check of RAID array md0 [300611.478132] md: data-check of RAID array md8 [301186.263972] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [301186.272501] Lustre: Skipped 5 previous similar messages [301412.862265] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbbc20b800, cur 1584864834 expire 1584864684 last 1584864607 [301412.882176] Lustre: Skipped 5 previous similar messages [302386.518008] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [302386.526535] Lustre: Skipped 5 previous similar messages [302612.865192] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffca9b42000, cur 1584866034 expire 1584865884 last 1584865807 [302612.885276] Lustre: Skipped 5 previous similar messages [303182.996138] LustreError: 49300:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli c2ca4c5a-e67e-4 claims 2519040 GRANT, real grant 1441792 [303586.645607] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [303586.654133] Lustre: Skipped 5 previous similar messages [303812.858892] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbec5fec00, cur 1584867234 expire 1584867084 last 1584867007 [303812.878776] Lustre: Skipped 5 previous similar messages [304786.772065] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [304786.780608] Lustre: Skipped 5 previous similar messages [305012.857044] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd3a2fc00, cur 1584868434 expire 1584868284 last 1584868207 [305012.876924] Lustre: Skipped 5 previous similar messages [305066.088571] LustreError: 48907:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 1f9d385a-f33f-4 claims 376832 GRANT, real grant 81920 [305986.898598] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [305986.907128] Lustre: Skipped 5 previous similar messages [306213.855225] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93011a800, cur 1584869635 expire 1584869485 last 1584869408 [306213.875113] Lustre: Skipped 5 previous similar messages [307185.999443] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [307186.007969] Lustre: Skipped 5 previous similar messages [307412.860101] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f1800, cur 1584870834 expire 1584870684 last 1584870607 [307412.879986] Lustre: Skipped 5 previous similar messages [307591.048661] LustreError: 49309:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 653b182b-aad1-4 claims 10579968 GRANT, real grant 2482176 [308386.409940] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [308386.418472] Lustre: Skipped 5 previous similar messages [308612.847842] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e901b000, cur 1584872034 expire 1584871884 last 1584871807 [308612.867752] Lustre: Skipped 5 previous similar messages [309586.634917] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [309586.643466] Lustre: Skipped 5 previous similar messages [309812.845868] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5971800, cur 1584873234 expire 1584873084 last 1584873007 [309812.865747] Lustre: Skipped 5 previous similar messages [310786.888521] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [310786.897058] Lustre: Skipped 5 previous similar messages [311013.841786] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9386d5c00, cur 1584874435 expire 1584874285 last 1584874208 [311013.861685] Lustre: Skipped 5 previous similar messages [311986.246422] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [311986.254954] Lustre: Skipped 5 previous similar messages [312212.840538] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f4000, cur 1584875634 expire 1584875484 last 1584875407 [312212.860550] Lustre: Skipped 5 previous similar messages [313186.499581] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [313186.508108] Lustre: Skipped 5 previous similar messages [313412.837070] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d0800, cur 1584876834 expire 1584876684 last 1584876607 [313412.856997] Lustre: Skipped 5 previous similar messages [313746.482749] LustreError: 49245:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 0271c837-bb5b-4 claims 1486848 GRANT, real grant 0 [314385.986155] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [314385.994683] Lustre: Skipped 5 previous similar messages [314612.836358] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e09c00, cur 1584878034 expire 1584877884 last 1584877807 [314612.856269] Lustre: Skipped 5 previous similar messages [315586.368969] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [315586.377499] Lustre: Skipped 5 previous similar messages [315812.833740] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5ad000, cur 1584879234 expire 1584879084 last 1584879007 [315812.853649] Lustre: Skipped 5 previous similar messages [316786.622388] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [316786.630918] Lustre: Skipped 5 previous similar messages [317012.830737] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbe7088800, cur 1584880434 expire 1584880284 last 1584880207 [317012.850624] Lustre: Skipped 5 previous similar messages [317825.827596] LustreError: 49399:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli eac7d7e6-7a39-4 claims 16752640 GRANT, real grant 10715136 [317986.190480] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [317986.199120] Lustre: Skipped 5 previous similar messages [318212.827879] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf3021c00, cur 1584881634 expire 1584881484 last 1584881407 [318212.847759] Lustre: Skipped 5 previous similar messages [318228.136166] LustreError: 49197:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2f954eed-4b6c-4 claims 1482752 GRANT, real grant 32768 [319186.488808] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [319186.497340] Lustre: Skipped 5 previous similar messages [319412.825112] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd932b2e000, cur 1584882834 expire 1584882684 last 1584882607 [319412.845026] Lustre: Skipped 5 previous similar messages [320386.869562] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [320386.878090] Lustre: Skipped 5 previous similar messages [320613.821265] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd49a268800, cur 1584884035 expire 1584883885 last 1584883808 [320613.841144] Lustre: Skipped 5 previous similar messages [321586.099192] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [321586.107719] Lustre: Skipped 5 previous similar messages [321812.819501] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931937400, cur 1584885234 expire 1584885084 last 1584885007 [321812.839389] Lustre: Skipped 5 previous similar messages [322786.481403] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [322786.489932] Lustre: Skipped 5 previous similar messages [323012.816232] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef1000, cur 1584886434 expire 1584886284 last 1584886207 [323012.836116] Lustre: Skipped 5 previous similar messages [323986.735613] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [323986.744140] Lustre: Skipped 5 previous similar messages [324212.815033] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb929c00, cur 1584887634 expire 1584887484 last 1584887407 [324212.834943] Lustre: Skipped 5 previous similar messages [325185.966719] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [325185.975252] Lustre: Skipped 5 previous similar messages [325412.813099] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef2c00, cur 1584888834 expire 1584888684 last 1584888607 [325412.832985] Lustre: Skipped 5 previous similar messages [325844.677457] LustreError: 48964:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 275b0d00-9e41-4 claims 393216 GRANT, real grant 155648 [326202.743139] LustreError: 49309:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 0271c837-bb5b-4 claims 28672 GRANT, real grant 0 [326386.348795] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [326386.357320] Lustre: Skipped 5 previous similar messages [326612.810265] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eaa84800, cur 1584890034 expire 1584889884 last 1584889807 [326612.830149] Lustre: Skipped 5 previous similar messages [327586.730789] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [327586.739321] Lustre: Skipped 5 previous similar messages [327812.808132] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb92a800, cur 1584891234 expire 1584891084 last 1584891007 [327812.828014] Lustre: Skipped 5 previous similar messages [328786.087756] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [328786.096283] Lustre: Skipped 5 previous similar messages [329012.805465] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf1f0cc00, cur 1584892434 expire 1584892284 last 1584892207 [329012.825345] Lustre: Skipped 5 previous similar messages [329986.341393] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [329986.349931] Lustre: Skipped 5 previous similar messages [330034.200170] LustreError: 49299:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 56bdeeef-d646-4 claims 1490944 GRANT, real grant 303104 [330212.810495] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7a3bd000, cur 1584893634 expire 1584893484 last 1584893407 [330212.830677] Lustre: Skipped 5 previous similar messages [330881.741155] LustreError: 49202:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 0271c837-bb5b-4 claims 286720 GRANT, real grant 28672 [331186.723666] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [331186.732192] Lustre: Skipped 5 previous similar messages [331412.801753] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf902e800, cur 1584894834 expire 1584894684 last 1584894607 [331412.821662] Lustre: Skipped 5 previous similar messages [332386.208443] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [332386.216978] Lustre: Skipped 5 previous similar messages [332612.795022] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fddde029c00, cur 1584896034 expire 1584895884 last 1584895807 [332612.814951] Lustre: Skipped 5 previous similar messages [333586.587915] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [333586.596443] Lustre: Skipped 5 previous similar messages [333812.794179] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd0beb000, cur 1584897234 expire 1584897084 last 1584897007 [333812.814060] Lustre: Skipped 5 previous similar messages [334785.944513] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [334785.953042] Lustre: Skipped 5 previous similar messages [335012.801611] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f574c00, cur 1584898434 expire 1584898284 last 1584898207 [335012.821494] Lustre: Skipped 5 previous similar messages [335986.325672] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [335986.334226] Lustre: Skipped 5 previous similar messages [336159.784928] Lustre: fir-OST0038: haven't heard from client 28cde8ea-b100-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd937e1f400, cur 1584899581 expire 1584899431 last 1584899354 [336159.804896] Lustre: Skipped 5 previous similar messages [337186.933482] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [337186.942011] Lustre: Skipped 11 previous similar messages [337412.784545] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bfa628400, cur 1584900834 expire 1584900684 last 1584900607 [337412.804443] Lustre: Skipped 11 previous similar messages [337640.787350] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80d800, cur 1584901062 expire 1584900912 last 1584900835 [337640.807266] Lustre: Skipped 2 previous similar messages [337818.780335] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eaa84400, cur 1584901240 expire 1584901090 last 1584901013 [338386.831668] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [338386.840206] Lustre: Skipped 5 previous similar messages [338613.777960] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934f8400, cur 1584902035 expire 1584901885 last 1584901808 [338613.797846] Lustre: Skipped 1 previous similar message [339586.189516] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [339586.198043] Lustre: Skipped 5 previous similar messages [339812.774246] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93132c800, cur 1584903234 expire 1584903084 last 1584903007 [339812.794128] Lustre: Skipped 5 previous similar messages [340786.570310] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [340786.578836] Lustre: Skipped 5 previous similar messages [341012.770375] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb809b7400, cur 1584904434 expire 1584904284 last 1584904207 [341012.790371] Lustre: Skipped 5 previous similar messages [341034.095701] LustreError: 49293:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 5682229c-f010-4 claims 16752640 GRANT, real grant 479232 [341054.469466] LustreError: 48944:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 39b158e5-2e75-4 claims 16752640 GRANT, real grant 1171456 [341985.942303] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [341985.950825] Lustre: Skipped 5 previous similar messages [342212.768798] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80a000, cur 1584905634 expire 1584905484 last 1584905407 [342212.788678] Lustre: Skipped 5 previous similar messages [342444.418887] LustreError: 49329:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 56bdeeef-d646-4 claims 1495040 GRANT, real grant 1490944 [343186.565551] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [343186.574081] Lustre: Skipped 5 previous similar messages [343412.778461] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934f9c00, cur 1584906834 expire 1584906684 last 1584906607 [343412.798372] Lustre: Skipped 5 previous similar messages [344385.923511] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [344385.932040] Lustre: Skipped 5 previous similar messages [344612.761553] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5adc00, cur 1584908034 expire 1584907884 last 1584907807 [344612.781435] Lustre: Skipped 5 previous similar messages [345166.789159] LustreError: 49271:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 0271c837-bb5b-4 claims 1462272 GRANT, real grant 286720 [345586.049757] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [345586.058286] Lustre: Skipped 5 previous similar messages [345812.759554] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5a9400, cur 1584909234 expire 1584909084 last 1584909007 [345812.779443] Lustre: Skipped 5 previous similar messages [346786.477700] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [346786.486244] Lustre: Skipped 5 previous similar messages [347012.757082] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf1f0ac00, cur 1584910434 expire 1584910284 last 1584910207 [347012.776988] Lustre: Skipped 5 previous similar messages [347986.814249] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [347986.822782] Lustre: Skipped 5 previous similar messages [348139.567008] LustreError: 67599:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 97171295-2f64-4 claims 3346432 GRANT, real grant 323584 [348213.756406] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf29b8000, cur 1584911635 expire 1584911485 last 1584911408 [348213.776287] Lustre: Skipped 5 previous similar messages [348772.042508] LustreError: 49073:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli f7fe261e-a413-4 claims 16752640 GRANT, real grant 1785856 [349186.199262] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [349186.207794] Lustre: Skipped 5 previous similar messages [349412.756466] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad921400, cur 1584912834 expire 1584912684 last 1584912607 [349412.776373] Lustre: Skipped 5 previous similar messages [350386.555599] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [350386.564132] Lustre: Skipped 5 previous similar messages [350612.756889] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5e74400, cur 1584914034 expire 1584913884 last 1584913807 [350612.776775] Lustre: Skipped 5 previous similar messages [351023.653595] LustreError: 49238:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli c7de84cf-8a38-4 claims 2867200 GRANT, real grant 802816 [351585.912866] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [351585.921396] Lustre: Skipped 5 previous similar messages [351812.748960] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934fdc00, cur 1584915234 expire 1584915084 last 1584915007 [351812.768843] Lustre: Skipped 5 previous similar messages [352786.293470] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [352786.302003] Lustre: Skipped 5 previous similar messages [353012.749247] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938549c00, cur 1584916434 expire 1584916284 last 1584916207 [353012.769170] Lustre: Skipped 5 previous similar messages [353986.676272] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [353986.684807] Lustre: Skipped 5 previous similar messages [354212.774710] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f1400, cur 1584917634 expire 1584917484 last 1584917407 [354212.794591] Lustre: Skipped 5 previous similar messages [354220.056837] LustreError: 50849:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 97171295-2f64-4 claims 4169728 GRANT, real grant 3379200 [355186.034199] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [355186.042728] Lustre: Skipped 5 previous similar messages [355212.713453] LustreError: 67479:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 17f2724d-4d81-4 claims 16752640 GRANT, real grant 10387456 [355412.750183] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b9e7de000, cur 1584918834 expire 1584918684 last 1584918607 [355412.770068] Lustre: Skipped 5 previous similar messages [356386.415256] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [356386.423805] Lustre: Skipped 5 previous similar messages [356612.765495] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f570000, cur 1584920034 expire 1584919884 last 1584919807 [356612.785383] Lustre: Skipped 5 previous similar messages [357586.797884] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [357586.806410] Lustre: Skipped 5 previous similar messages [357813.746822] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f3800, cur 1584921235 expire 1584921085 last 1584921008 [357813.766702] Lustre: Skipped 5 previous similar messages [358786.155604] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [358786.164133] Lustre: Skipped 5 previous similar messages [359012.732340] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bce6d1400, cur 1584922434 expire 1584922284 last 1584922207 [359012.752224] Lustre: Skipped 5 previous similar messages [359497.755026] LustreError: 49270:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli b4929648-8e2a-4 claims 180224 GRANT, real grant 0 [359521.350437] LustreError: 49187:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 5682229c-f010-4 claims 159744 GRANT, real grant 73728 [359780.365693] LustreError: 49275:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 322961a3-22bb-4 claims 2490368 GRANT, real grant 1441792 [359986.408922] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [359986.417499] Lustre: Skipped 5 previous similar messages [360212.728524] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf57f7c00, cur 1584923634 expire 1584923484 last 1584923407 [360212.748423] Lustre: Skipped 5 previous similar messages [361186.789259] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [361186.797787] Lustre: Skipped 5 previous similar messages [361413.736454] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00befa7d000, cur 1584924835 expire 1584924685 last 1584924608 [361413.756366] Lustre: Skipped 5 previous similar messages [362386.906084] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [362386.914619] Lustre: Skipped 5 previous similar messages [362612.724366] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbe7089000, cur 1584926034 expire 1584925884 last 1584925807 [362612.744251] Lustre: Skipped 5 previous similar messages [363586.015956] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [363586.024501] Lustre: Skipped 5 previous similar messages [363812.720205] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bfa66e000, cur 1584927234 expire 1584927084 last 1584927007 [363812.740090] Lustre: Skipped 5 previous similar messages [364332.421863] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584927746/real 1584927746] req@ffffa00062bf1680 x1661536658534336/t0(0) o104->fir-OST0032@10.50.9.37@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1584927753 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [364333.187857] Lustre: 48980:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584927747/real 1584927747] req@ffff9fcbd36aba80 x1661536658780736/t0(0) o104->fir-OST0036@10.50.9.37@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1584927754 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [364339.448847] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584927753/real 1584927753] req@ffffa00062bf1680 x1661536658534336/t0(0) o104->fir-OST0032@10.50.9.37@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1584927760 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [364339.476262] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [364346.485827] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584927760/real 1584927760] req@ffffa00062bf1680 x1661536658534336/t0(0) o104->fir-OST0032@10.50.9.37@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1584927767 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [364346.513183] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 7 previous similar messages [364353.522811] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584927767/real 1584927767] req@ffffa00062bf1680 x1661536658534336/t0(0) o104->fir-OST0032@10.50.9.37@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1584927774 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [364353.550143] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 8 previous similar messages [364361.585771] Lustre: 48846:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584927775/real 1584927775] req@ffff9fcbd9ad5100 x1661536659158272/t0(0) o104->fir-OST0032@10.50.9.37@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1584927782 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [364361.613142] Lustre: 48846:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 15 previous similar messages [364381.559738] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1584927795/real 1584927795] req@ffffa00062bf1680 x1661536658534336/t0(0) o104->fir-OST0032@10.50.9.37@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1584927802 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [364381.587105] Lustre: 48912:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 19 previous similar messages [364387.727913] LustreError: 48853:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.50.9.37@o2ib2) failed to reply to blocking AST (req@ffff9ff412be1680 x1661536658983424 status 0 rc -5), evict it ns: filter-fir-OST0036_UUID lock: ffff9fcbfe3f3a80/0xf3793efb04820cb2 lrc: 4/0,0 mode: PR/PR res: [0x1080000402:0xe4b7fd:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->134217727) flags: 0x60000400000020 nid: 10.50.9.37@o2ib2 remote: 0x650410efd46f1689 expref: 19 pid: 48847 timeout: 364532 lvb_type: 1 [364387.727919] LustreError: 138-a: fir-OST0036: A client on nid 10.50.9.37@o2ib2 was evicted due to a lock blocking callback time out: rc -5 [364387.727945] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 62s: evicting client at 10.50.9.37@o2ib2 ns: filter-fir-OST0036_UUID lock: ffff9fde2809cec0/0xf3793efb0486efa5 lrc: 3/0,0 mode: PR/PR res: [0x1080000402:0xe4b92f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->134217727) flags: 0x60000400000020 nid: 10.50.9.37@o2ib2 remote: 0x650410efd46fceea expref: 16 pid: 48826 timeout: 0 lvb_type: 1 [364387.826707] LustreError: 48853:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) Skipped 7 previous similar messages [364451.480248] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [364451.488862] Lustre: Skipped 9 previous similar messages [364917.717685] Lustre: fir-OST0032: haven't heard from client bc12991d-b630-4 (at 10.50.6.22@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e19b800, cur 1584928339 expire 1584928189 last 1584928112 [364917.737656] Lustre: Skipped 11 previous similar messages [365194.204960] Lustre: fir-OST0032: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [365194.213493] Lustre: Skipped 5 previous similar messages [365950.866162] LustreError: 49251:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli b6453fa2-7e92-4 claims 90112 GRANT, real grant 32768 [365986.010456] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [365986.018987] Lustre: Skipped 1 previous similar message [366212.718323] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd0bea000, cur 1584929634 expire 1584929484 last 1584929407 [366212.738229] Lustre: Skipped 11 previous similar messages [366440.712395] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf721c000, cur 1584929862 expire 1584929712 last 1584929635 [366440.732278] Lustre: Skipped 2 previous similar messages [366621.754389] Lustre: fir-OST0032: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9192400, cur 1584930043 expire 1584929893 last 1584929816 [367186.393348] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [367186.401879] Lustre: Skipped 5 previous similar messages [367412.710088] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bcef7f800, cur 1584930834 expire 1584930684 last 1584930607 [367412.729976] Lustre: Skipped 1 previous similar message [368386.774310] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [368386.782842] Lustre: Skipped 5 previous similar messages [368613.724561] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febdba9c800, cur 1584932035 expire 1584931885 last 1584931808 [368613.744441] Lustre: Skipped 5 previous similar messages [369586.132426] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [369586.140955] Lustre: Skipped 5 previous similar messages [369812.705763] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf1f0fc00, cur 1584933234 expire 1584933084 last 1584933007 [369812.725648] Lustre: Skipped 5 previous similar messages [370265.770625] LustreError: 49202:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3df0b565-ed22-4 claims 106496 GRANT, real grant 32768 [370786.637494] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [370786.646025] Lustre: Skipped 5 previous similar messages [371012.717147] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd1f1500400, cur 1584934434 expire 1584934284 last 1584934207 [371012.737057] Lustre: Skipped 5 previous similar messages [371986.130162] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [371986.138709] Lustre: Skipped 5 previous similar messages [372212.717378] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5f61400, cur 1584935634 expire 1584935484 last 1584935407 [372212.737289] Lustre: Skipped 5 previous similar messages [372250.217702] LustreError: 49009:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3df0b565-ed22-4 claims 110592 GRANT, real grant 106496 [373186.511806] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [373186.520332] Lustre: Skipped 5 previous similar messages [373412.699873] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fddde02d400, cur 1584936834 expire 1584936684 last 1584936607 [373412.719778] Lustre: Skipped 5 previous similar messages [374386.184785] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [374386.193315] Lustre: Skipped 5 previous similar messages [374612.699162] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b9e7db000, cur 1584938034 expire 1584937884 last 1584937807 [374612.719065] Lustre: Skipped 5 previous similar messages [375586.508328] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [375586.516858] Lustre: Skipped 5 previous similar messages [375812.700140] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf5fed000, cur 1584939234 expire 1584939084 last 1584939007 [375812.720054] Lustre: Skipped 5 previous similar messages [376786.249260] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [376786.257791] Lustre: Skipped 5 previous similar messages [377012.690806] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b83b4fc00, cur 1584940434 expire 1584940284 last 1584940207 [377012.710711] Lustre: Skipped 5 previous similar messages [377153.857235] LustreError: 49314:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli d0ed2bdc-7329-4 claims 28672 GRANT, real grant 0 [377985.868725] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [377985.877256] Lustre: Skipped 5 previous similar messages [378212.691395] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb66b74000, cur 1584941634 expire 1584941484 last 1584941407 [378212.711302] Lustre: Skipped 5 previous similar messages [379186.437747] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [379186.446273] Lustre: Skipped 5 previous similar messages [379412.685953] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b58c00, cur 1584942834 expire 1584942684 last 1584942607 [379412.705869] Lustre: Skipped 5 previous similar messages [380385.923979] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [380385.932511] Lustre: Skipped 5 previous similar messages [380488.443859] LustreError: 48964:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli f5a2750a-4518-4 claims 131072 GRANT, real grant 110592 [380612.685842] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf0eea400, cur 1584944034 expire 1584943884 last 1584943807 [380612.705724] Lustre: Skipped 5 previous similar messages [380761.799047] LustreError: 49312:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 6479814a-e93b-4 claims 28672 GRANT, real grant 0 [381586.113491] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [381586.122015] Lustre: Skipped 5 previous similar messages [381812.696449] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bee9ca800, cur 1584945234 expire 1584945084 last 1584945007 [381812.716354] Lustre: Skipped 5 previous similar messages [382786.365790] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [382786.374319] Lustre: Skipped 5 previous similar messages [383012.677093] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80cc00, cur 1584946434 expire 1584946284 last 1584946207 [383012.697006] Lustre: Skipped 5 previous similar messages [383986.746830] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [383986.755397] Lustre: Skipped 5 previous similar messages [384213.692792] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00befa79800, cur 1584947635 expire 1584947485 last 1584947408 [384213.712727] Lustre: Skipped 5 previous similar messages [385186.105393] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [385186.113925] Lustre: Skipped 5 previous similar messages [385412.673459] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f574c00, cur 1584948834 expire 1584948684 last 1584948607 [385412.693420] Lustre: Skipped 5 previous similar messages [386386.230331] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [386386.238864] Lustre: Skipped 5 previous similar messages [386420.769231] LustreError: 49091:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli dff180c1-de42-4 claims 229376 GRANT, real grant 155648 [386612.669297] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931932800, cur 1584950034 expire 1584949884 last 1584949807 [386612.689206] Lustre: Skipped 5 previous similar messages [387586.356182] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [387586.364710] Lustre: Skipped 5 previous similar messages [387812.667778] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf721b000, cur 1584951234 expire 1584951084 last 1584951007 [387812.687687] Lustre: Skipped 5 previous similar messages [388786.483135] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [388786.491664] Lustre: Skipped 5 previous similar messages [389012.666062] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbec5fb400, cur 1584952434 expire 1584952284 last 1584952207 [389012.685968] Lustre: Skipped 5 previous similar messages [389986.608195] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [389986.616722] Lustre: Skipped 5 previous similar messages [390212.660259] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ebaad400, cur 1584953634 expire 1584953484 last 1584953407 [390212.680238] Lustre: Skipped 5 previous similar messages [390216.410587] LustreError: 49377:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 00d873ba-49ea-4 claims 143360 GRANT, real grant 61440 [390603.326202] LustreError: 49352:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli ec565b6c-3dc6-4 claims 970752 GRANT, real grant 794624 [390667.700400] LustreError: 102174:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 86db2c50-4e1f-4 claims 319488 GRANT, real grant 110592 [391186.029207] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [391186.037742] Lustre: Skipped 5 previous similar messages [391412.656250] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00a38a54c00, cur 1584954834 expire 1584954684 last 1584954607 [391412.676159] Lustre: Skipped 5 previous similar messages [391434.229517] LustreError: 49268:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 56bdeeef-d646-4 claims 1523712 GRANT, real grant 1495040 [392386.282969] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [392386.291501] Lustre: Skipped 5 previous similar messages [392612.655387] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf902dc00, cur 1584956034 expire 1584955884 last 1584955807 [392612.675271] Lustre: Skipped 5 previous similar messages [393586.153029] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [393586.161556] Lustre: Skipped 5 previous similar messages [393812.655435] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef3000, cur 1584957234 expire 1584957084 last 1584957007 [393812.675318] Lustre: Skipped 5 previous similar messages [394786.278116] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [394786.286660] Lustre: Skipped 5 previous similar messages [395012.649862] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb66b74c00, cur 1584958434 expire 1584958284 last 1584958207 [395012.669762] Lustre: Skipped 5 previous similar messages [395221.276705] LustreError: 49148:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 67876c18-c2e3-4 claims 970752 GRANT, real grant 122880 [395255.321908] LustreError: 49338:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli dff180c1-de42-4 claims 1499136 GRANT, real grant 229376 [395406.289516] LustreError: 49273:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 2342912 GRANT, real grant 1691648 [395986.404131] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [395986.412665] Lustre: Skipped 5 previous similar messages [396212.650653] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5f62000, cur 1584959634 expire 1584959484 last 1584959407 [396212.670552] Lustre: Skipped 5 previous similar messages [397186.529948] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [397186.538498] Lustre: Skipped 5 previous similar messages [397412.644770] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd829f400, cur 1584960834 expire 1584960684 last 1584960607 [397412.664673] Lustre: Skipped 5 previous similar messages [398386.655877] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [398386.664402] Lustre: Skipped 5 previous similar messages [398613.642116] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ac44c00, cur 1584962035 expire 1584961885 last 1584961808 [398613.661997] Lustre: Skipped 5 previous similar messages [399330.644262] Lustre: fir-OST0038: haven't heard from client c6acd997-7594-4 (at 10.50.9.72@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb95b04800, cur 1584962752 expire 1584962602 last 1584962525 [399330.664288] Lustre: Skipped 5 previous similar messages [399585.821801] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [399585.830360] Lustre: Skipped 5 previous similar messages [400041.639892] Lustre: fir-OST0034: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf57f6c00, cur 1584963463 expire 1584963313 last 1584963236 [400041.659817] Lustre: Skipped 8 previous similar messages [400067.799461] LustreError: 48896:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3da00d75-6c3b-4 claims 139264 GRANT, real grant 106496 [400785.948050] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [400785.956580] Lustre: Skipped 5 previous similar messages [401012.639442] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a4c00, cur 1584964434 expire 1584964284 last 1584964207 [401012.659330] Lustre: Skipped 2 previous similar messages [401986.330172] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [401986.338701] Lustre: Skipped 5 previous similar messages [402212.634585] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fddde02ac00, cur 1584965634 expire 1584965484 last 1584965407 [402212.654468] Lustre: Skipped 5 previous similar messages [403186.712846] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [403186.721379] Lustre: Skipped 5 previous similar messages [403413.640651] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938eac400, cur 1584966835 expire 1584966685 last 1584966608 [403413.660556] Lustre: Skipped 5 previous similar messages [404385.814733] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [404385.823269] Lustre: Skipped 5 previous similar messages [404612.632866] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5f60000, cur 1584968034 expire 1584967884 last 1584967807 [404612.652789] Lustre: Skipped 5 previous similar messages [404863.617242] LustreError: 49256:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 35cd2c2f-e02c-4 claims 16752640 GRANT, real grant 10579968 [405585.939507] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [405585.948030] Lustre: Skipped 5 previous similar messages [405812.627183] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9357ef800, cur 1584969234 expire 1584969084 last 1584969007 [405812.647113] Lustre: Skipped 5 previous similar messages [406786.065991] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [406786.074526] Lustre: Skipped 5 previous similar messages [407012.629477] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf91c4800, cur 1584970434 expire 1584970284 last 1584970207 [407012.649409] Lustre: Skipped 5 previous similar messages [407386.397293] LustreError: 49354:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 23b73773-98ec-4 claims 1445888 GRANT, real grant 380928 [407986.190287] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [407986.198876] Lustre: Skipped 5 previous similar messages [408212.620767] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e52d7800, cur 1584971634 expire 1584971484 last 1584971407 [408212.640685] Lustre: Skipped 5 previous similar messages [409186.316660] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [409186.325193] Lustre: Skipped 5 previous similar messages [409412.616649] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf1f0fc00, cur 1584972834 expire 1584972684 last 1584972607 [409412.636532] Lustre: Skipped 5 previous similar messages [410386.698070] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [410386.706599] Lustre: Skipped 5 previous similar messages [410613.623817] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a7800, cur 1584974035 expire 1584973885 last 1584973808 [410613.643702] Lustre: Skipped 5 previous similar messages [411585.798621] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [411585.807151] Lustre: Skipped 5 previous similar messages [411812.615205] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5ad400, cur 1584975234 expire 1584975084 last 1584975007 [411812.635200] Lustre: Skipped 5 previous similar messages [412785.925155] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [412785.933682] Lustre: Skipped 5 previous similar messages [413012.609149] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf1f0e800, cur 1584976434 expire 1584976284 last 1584976207 [413012.629079] Lustre: Skipped 5 previous similar messages [413463.060338] LustreError: 49276:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 56bdeeef-d646-4 claims 1576960 GRANT, real grant 1523712 [413986.306307] Lustre: fir-OST0038: Connection restored to 417f1855-bc48-4 (at 10.50.9.7@o2ib2) [413986.314837] Lustre: Skipped 5 previous similar messages [414212.605856] Lustre: fir-OST0038: haven't heard from client 417f1855-bc48-4 (at 10.50.9.7@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf91c7800, cur 1584977634 expire 1584977484 last 1584977407 [414212.625736] Lustre: Skipped 5 previous similar messages [414587.723834] Lustre: fir-OST0030: Connection restored to bd34050e-30d7-4 (at 10.50.10.66@o2ib2) [414587.732539] Lustre: Skipped 50 previous similar messages [416522.677623] LustreError: 49282:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 665344bc-9abf-4 claims 192512 GRANT, real grant 28672 [416906.651574] LustreError: 56061:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a67a88ce-e5ae-4 claims 12296192 GRANT, real grant 11575296 [418484.596262] Lustre: fir-OST0038: haven't heard from client 7afa46a0-f8e2-4 (at 10.50.17.42@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb91c9c800, cur 1584981906 expire 1584981756 last 1584981679 [418484.616318] Lustre: Skipped 75 previous similar messages [418508.361022] Lustre: fir-OST0030: Connection restored to 7afa46a0-f8e2-4 (at 10.50.17.42@o2ib2) [418508.369721] Lustre: Skipped 53 previous similar messages [419166.095297] LustreError: 48335:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 23b73773-98ec-4 claims 1454080 GRANT, real grant 1445888 [420133.228490] LustreError: 49076:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2f954eed-4b6c-4 claims 16752640 GRANT, real grant 1482752 [421378.751208] LustreError: 102175:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 0ce69c96-29d6-4 claims 1495040 GRANT, real grant 589824 [421518.108001] LustreError: 99541:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 67876c18-c2e3-4 claims 1519616 GRANT, real grant 970752 [428980.444217] LustreError: 56061:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli e0e2514e-9993-4 claims 1331200 GRANT, real grant 28672 [429642.675620] LustreError: 49361:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 8d039ea4-d34b-4 claims 110592 GRANT, real grant 0 [432309.209333] LustreError: 49198:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 00d873ba-49ea-4 claims 229376 GRANT, real grant 143360 [432337.563308] Lustre: fir-OST0036: haven't heard from client ce67c0db-bb4c-4 (at 10.50.15.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d6b0400, cur 1584995759 expire 1584995609 last 1584995532 [432337.583277] Lustre: Skipped 5 previous similar messages [433680.560085] Lustre: fir-OST0030: haven't heard from client 2e6641b9-e38a-4 (at 10.50.12.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80eb3c00, cur 1584997102 expire 1584996952 last 1584996875 [433680.580081] Lustre: Skipped 17 previous similar messages [433683.559616] Lustre: fir-OST0032: haven't heard from client 2e6641b9-e38a-4 (at 10.50.12.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e4cf400, cur 1584997105 expire 1584996955 last 1584996878 [433697.559740] Lustre: fir-OST0038: haven't heard from client 2e6641b9-e38a-4 (at 10.50.12.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf75df000, cur 1584997119 expire 1584996969 last 1584996892 [433697.579763] Lustre: Skipped 2 previous similar messages [433756.561748] Lustre: fir-OST0030: haven't heard from client 6d03d9a7-b1f2-4 (at 10.50.12.9@o2ib2) in 188 seconds. I think it's dead, and I am evicting it. exp ffff9feb80edb000, cur 1584997178 expire 1584997028 last 1584996990 [433756.581765] Lustre: Skipped 1 previous similar message [433773.559651] Lustre: fir-OST0038: haven't heard from client 6d03d9a7-b1f2-4 (at 10.50.12.9@o2ib2) in 195 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfeaf3800, cur 1584997195 expire 1584997045 last 1584997000 [433773.579653] Lustre: Skipped 1 previous similar message [433805.564906] Lustre: fir-OST0036: haven't heard from client 6d03d9a7-b1f2-4 (at 10.50.12.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d6b7000, cur 1584997227 expire 1584997077 last 1584997000 [433810.417227] Lustre: fir-OST0030: Connection restored to 6d03d9a7-b1f2-4 (at 10.50.12.9@o2ib2) [433810.425847] Lustre: Skipped 5 previous similar messages [434137.378748] Lustre: fir-OST0030: Connection restored to 4304d775-beb4-4 (at 10.50.13.7@o2ib2) [434137.387404] Lustre: Skipped 5 previous similar messages [434291.558401] Lustre: fir-OST0030: haven't heard from client 645377df-0a0a-4 (at 10.50.14.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80f7b400, cur 1584997713 expire 1584997563 last 1584997486 [434291.578510] Lustre: Skipped 2 previous similar messages [434781.616069] Lustre: fir-OST0030: Connection restored to f8bb9b7d-b961-4 (at 10.50.14.9@o2ib2) [434781.624684] Lustre: Skipped 5 previous similar messages [434842.556838] Lustre: fir-OST0032: haven't heard from client f8bb9b7d-b961-4 (at 10.50.14.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e4c9c00, cur 1584998264 expire 1584998114 last 1584998037 [434842.576808] Lustre: Skipped 5 previous similar messages [434978.056771] Lustre: fir-OST0030: Connection restored to 2e6641b9-e38a-4 (at 10.50.12.8@o2ib2) [434978.065575] Lustre: Skipped 5 previous similar messages [435330.556509] Lustre: fir-OST0032: haven't heard from client c05f4f30-e5ee-4 (at 10.50.12.10@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e651800, cur 1584998752 expire 1584998602 last 1584998525 [435330.576594] Lustre: Skipped 5 previous similar messages [435341.664893] Lustre: fir-OST0030: Connection restored to c05f4f30-e5ee-4 (at 10.50.12.10@o2ib2) [435341.673615] Lustre: Skipped 5 previous similar messages [435594.695768] Lustre: fir-OST0030: Connection restored to 645377df-0a0a-4 (at 10.50.14.10@o2ib2) [435594.704468] Lustre: Skipped 5 previous similar messages [437440.082241] Lustre: fir-OST0030: Connection restored to 50a6df69-dfc3-4 (at 10.50.12.6@o2ib2) [437440.090864] Lustre: Skipped 5 previous similar messages [437469.553175] Lustre: fir-OST0038: haven't heard from client 50a6df69-dfc3-4 (at 10.50.12.6@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbcf264c00, cur 1585000891 expire 1585000741 last 1585000664 [437469.573186] Lustre: Skipped 5 previous similar messages [437811.549748] Lustre: fir-OST0036: haven't heard from client 2d9a52b2-e42c-4 (at 10.50.12.11@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7dce7400, cur 1585001233 expire 1585001083 last 1585001006 [437811.569827] Lustre: Skipped 5 previous similar messages [437826.910703] Lustre: fir-OST0030: Connection restored to 2d9a52b2-e42c-4 (at 10.50.12.11@o2ib2) [437826.919402] Lustre: Skipped 5 previous similar messages [438604.548338] Lustre: fir-OST003a: haven't heard from client 2422a71a-3ef8-4 (at 10.50.14.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7efcbc00, cur 1585002026 expire 1585001876 last 1585001799 [438604.568420] Lustre: Skipped 5 previous similar messages [438639.979010] Lustre: fir-OST0030: Connection restored to 8530b1fc-5bf1-4 (at 10.50.14.13@o2ib2) [438639.987720] Lustre: Skipped 5 previous similar messages [439123.876344] LustreError: 67599:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 8d039ea4-d34b-4 claims 16752640 GRANT, real grant 110592 [439270.919310] Lustre: fir-OST0030: Connection restored to 38135a83-ec5c-4 (at 10.50.12.5@o2ib2) [439270.927924] Lustre: Skipped 5 previous similar messages [439306.545449] Lustre: fir-OST0032: haven't heard from client 38135a83-ec5c-4 (at 10.50.12.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e651400, cur 1585002728 expire 1585002578 last 1585002501 [439306.565466] Lustre: Skipped 17 previous similar messages [439756.939474] Lustre: fir-OST0030: Connection restored to e63c4438-c6e3-4 (at 10.50.15.8@o2ib2) [439756.948097] Lustre: Skipped 5 previous similar messages [439822.935344] Lustre: fir-OST0030: Connection restored to 72c8f4c4-9808-4 (at 10.50.13.4@o2ib2) [439822.943963] Lustre: Skipped 5 previous similar messages [439877.925628] Lustre: fir-OST0030: Connection restored to 46aae41a-dd38-4 (at 10.50.15.1@o2ib2) [439877.934269] Lustre: Skipped 5 previous similar messages [439895.947346] Lustre: fir-OST0030: Connection restored to 1c46010b-e86f-4 (at 10.50.14.3@o2ib2) [439895.955965] Lustre: Skipped 5 previous similar messages [439912.346602] Lustre: fir-OST0030: Connection restored to 2d2dde34-cdb2-4 (at 10.50.15.7@o2ib2) [439912.355226] Lustre: Skipped 11 previous similar messages [439913.545573] Lustre: fir-OST003a: haven't heard from client b2463005-ec38-4 (at 10.50.16.2@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7eed7800, cur 1585003335 expire 1585003185 last 1585003108 [439913.565576] Lustre: Skipped 5 previous similar messages [439916.559910] Lustre: fir-OST0032: haven't heard from client 0f689ff7-6991-4 (at 10.50.14.4@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e219000, cur 1585003338 expire 1585003188 last 1585003111 [439916.579899] Lustre: Skipped 59 previous similar messages [439946.398808] Lustre: fir-OST0030: Connection restored to d66e7d02-84db-4 (at 10.49.28.11@o2ib1) [439946.407541] Lustre: Skipped 11 previous similar messages [440012.411421] Lustre: fir-OST0030: Connection restored to ccea6ca8-94f6-4 (at 10.50.15.3@o2ib2) [440012.420036] Lustre: Skipped 17 previous similar messages [440813.631496] LustreError: 56063:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 67876c18-c2e3-4 claims 1523712 GRANT, real grant 1519616 [440830.842490] Lustre: fir-OST0030: Connection restored to 0c02c6b3-fbbf-4 (at 10.50.13.14@o2ib2) [440830.851206] Lustre: Skipped 16 previous similar messages [441085.940694] Lustre: fir-OST0030: Connection restored to fd153b9f-50b1-4 (at 10.50.13.6@o2ib2) [441085.949311] Lustre: Skipped 5 previous similar messages [441102.019861] Lustre: fir-OST0030: Connection restored to 842a3b99-802c-4 (at 10.50.14.7@o2ib2) [441102.028477] Lustre: Skipped 5 previous similar messages [441149.897838] Lustre: fir-OST0030: Connection restored to 7c9c28a0-1550-4 (at 10.50.15.11@o2ib2) [441149.906541] Lustre: Skipped 29 previous similar messages [441220.845521] LustreError: 49266:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 7940c6f6-66d7-4 claims 1560576 GRANT, real grant 57344 [441657.849930] LustreError: 49326:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cbf4c94a-83d4-4 claims 28672 GRANT, real grant 24576 [441884.758795] Lustre: fir-OST0030: Connection restored to e0b3c403-4bb2-4 (at 10.50.14.6@o2ib2) [441884.767410] Lustre: Skipped 11 previous similar messages [442540.959385] Lustre: fir-OST0030: Connection restored to 0f689ff7-6991-4 (at 10.50.14.4@o2ib2) [442540.968001] Lustre: Skipped 5 previous similar messages [442561.496739] Lustre: fir-OST0030: Connection restored to ef0fa67c-0910-4 (at 10.50.14.2@o2ib2) [442561.505354] Lustre: Skipped 5 previous similar messages [443871.906939] Lustre: fir-OST0030: Connection restored to 482fd27b-a473-4 (at 10.50.13.11@o2ib2) [443871.915640] Lustre: Skipped 5 previous similar messages [443875.545057] Lustre: fir-OST0034: haven't heard from client 482fd27b-a473-4 (at 10.50.13.11@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb8164bc00, cur 1585007297 expire 1585007147 last 1585007070 [443875.565119] Lustre: Skipped 11 previous similar messages [444200.462140] LustreError: 49243:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3e4eec45-c061-4 claims 16752640 GRANT, real grant 1253376 [444612.534451] Lustre: fir-OST0034: haven't heard from client e1991612-ee0d-4 (at 10.50.14.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd49a268000, cur 1585008034 expire 1585007884 last 1585007807 [444612.554508] Lustre: Skipped 5 previous similar messages [444798.535064] Lustre: fir-OST003a: haven't heard from client fbd4143c-cb04-4 (at 10.50.13.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ebb8c00, cur 1585008220 expire 1585008070 last 1585007993 [444798.555053] Lustre: Skipped 5 previous similar messages [445326.532933] Lustre: fir-OST003a: haven't heard from client 1ce69533-b12b-4 (at 10.49.29.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ef75400, cur 1585008748 expire 1585008598 last 1585008521 [445326.552933] Lustre: Skipped 5 previous similar messages [445735.563218] LustreError: 48908:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 7940c6f6-66d7-4 claims 16134144 GRANT, real grant 1560576 [445747.532342] Lustre: fir-OST0036: haven't heard from client 2ac9ee52-f29e-4 (at 10.49.29.1@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d701800, cur 1585009169 expire 1585009019 last 1585008942 [445747.552336] Lustre: Skipped 5 previous similar messages [445847.782143] Lustre: fir-OST0030: Connection restored to 8530b1fc-5bf1-4 (at 10.50.14.13@o2ib2) [445847.790845] Lustre: Skipped 5 previous similar messages [446052.530605] LustreError: 51669:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3df0b565-ed22-4 claims 327680 GRANT, real grant 110592 [446206.530995] Lustre: fir-OST003a: haven't heard from client 4c86e85e-ac4f-4 (at 10.49.28.1@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7efc9c00, cur 1585009628 expire 1585009478 last 1585009401 [446206.550962] Lustre: Skipped 5 previous similar messages [446230.966329] Lustre: fir-OST0032: Connection restored to e4a1516e-1b3e-4 (at 10.50.12.15@o2ib2) [446230.975032] Lustre: Skipped 5 previous similar messages [446685.539975] Lustre: fir-OST0030: Connection restored to 1ce69533-b12b-4 (at 10.49.29.3@o2ib1) [446685.548605] Lustre: Skipped 5 previous similar messages [446748.730980] Lustre: fir-OST0032: Connection restored to fbd4143c-cb04-4 (at 10.50.13.8@o2ib2) [446748.739594] Lustre: Skipped 5 previous similar messages [446881.967159] Lustre: fir-OST0030: Connection restored to 2672fd12-19fb-4 (at 10.50.12.7@o2ib2) [446881.975903] Lustre: Skipped 4 previous similar messages [447094.863464] Lustre: fir-OST0030: Connection restored to 2ac9ee52-f29e-4 (at 10.49.29.1@o2ib1) [447094.872078] Lustre: Skipped 6 previous similar messages [447585.851732] Lustre: fir-OST0030: Connection restored to 4c86e85e-ac4f-4 (at 10.49.28.1@o2ib1) [447585.860352] Lustre: Skipped 5 previous similar messages [448023.648141] Lustre: fir-OST0032: Connection restored to 6627faaf-f7b9-4 (at 10.50.12.17@o2ib2) [448023.656849] Lustre: Skipped 4 previous similar messages [448426.730122] LustreError: 51669:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli fa1e1112-266d-4 claims 28672 GRANT, real grant 20480 [449838.890096] Lustre: fir-OST0030: Connection restored to ab87f0a5-0357-4 (at 10.49.29.5@o2ib1) [449838.898711] Lustre: Skipped 5 previous similar messages [450903.279589] Lustre: fir-OST0030: Connection restored to b8ff2e89-6825-4 (at 10.50.13.2@o2ib2) [450903.288205] Lustre: Skipped 5 previous similar messages [451064.522142] Lustre: fir-OST0034: haven't heard from client 3e79a8aa-e20a-4 (at 10.50.12.16@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb814dec00, cur 1585014486 expire 1585014336 last 1585014259 [451064.542206] Lustre: Skipped 5 previous similar messages [452279.404295] Lustre: fir-OST0030: Connection restored to 3e79a8aa-e20a-4 (at 10.50.12.16@o2ib2) [452279.413097] Lustre: Skipped 5 previous similar messages [452531.518528] Lustre: fir-OST0034: haven't heard from client 5d22cf6d-6c39-4 (at 10.49.29.7@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb81674c00, cur 1585015953 expire 1585015803 last 1585015726 [452531.538526] Lustre: Skipped 5 previous similar messages [453893.236061] Lustre: fir-OST0030: Connection restored to 5d22cf6d-6c39-4 (at 10.49.29.7@o2ib1) [453893.244701] Lustre: Skipped 5 previous similar messages [455517.510801] Lustre: fir-OST0032: haven't heard from client b4f8cb5a-edfb-4 (at 10.50.13.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e653c00, cur 1585018939 expire 1585018789 last 1585018712 [455517.530774] Lustre: Skipped 5 previous similar messages [456757.754214] Lustre: fir-OST0030: Connection restored to b4f8cb5a-edfb-4 (at 10.50.13.3@o2ib2) [456757.762827] Lustre: Skipped 5 previous similar messages [457077.255901] LustreError: 91449:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 00c7f158-cc8b-4 claims 28672 GRANT, real grant 8192 [457331.508630] Lustre: fir-OST0030: haven't heard from client 35cd2c2f-e02c-4 (at 10.50.12.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80f7dc00, cur 1585020753 expire 1585020603 last 1585020526 [457331.529439] Lustre: Skipped 5 previous similar messages [457753.327755] LustreError: 49074:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli e19e4c6e-1211-4 claims 16752640 GRANT, real grant 10936320 [458401.504703] Lustre: fir-OST0032: haven't heard from client 13489a82-b9be-4 (at 10.50.13.1@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e1c8400, cur 1585021823 expire 1585021673 last 1585021596 [458401.524719] Lustre: Skipped 5 previous similar messages [458544.955528] Lustre: fir-OST0032: Connection restored to 35cd2c2f-e02c-4 (at 10.50.12.13@o2ib2) [458544.964235] Lustre: Skipped 5 previous similar messages [459348.501935] Lustre: fir-OST003a: haven't heard from client 37a9513e-0a78-4 (at 10.50.12.14@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ef44800, cur 1585022770 expire 1585022620 last 1585022543 [459348.521995] Lustre: Skipped 5 previous similar messages [459577.502206] Lustre: fir-OST0034: haven't heard from client 8db8300c-7159-4 (at 10.50.14.14@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb814d6800, cur 1585022999 expire 1585022849 last 1585022772 [459577.522259] Lustre: Skipped 17 previous similar messages [459598.428476] Lustre: fir-OST0030: Connection restored to 13489a82-b9be-4 (at 10.50.13.1@o2ib2) [459598.437125] Lustre: Skipped 5 previous similar messages [460259.500663] Lustre: fir-OST0034: haven't heard from client efbc6add-d7ba-4 (at 10.50.12.4@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7f812400, cur 1585023681 expire 1585023531 last 1585023454 [460259.520720] Lustre: Skipped 5 previous similar messages [460564.080980] Lustre: fir-OST0030: Connection restored to 37a9513e-0a78-4 (at 10.50.12.14@o2ib2) [460564.089688] Lustre: Skipped 5 previous similar messages [460680.992670] Lustre: fir-OST0032: Connection restored to 91cab27d-d429-4 (at 10.49.29.2@o2ib1) [460680.992671] Lustre: fir-OST003a: Connection restored to 91cab27d-d429-4 (at 10.49.29.2@o2ib1) [460681.009902] Lustre: Skipped 3 previous similar messages [460693.499385] Lustre: fir-OST0034: haven't heard from client 90f2fa35-5a19-4 (at 10.49.29.8@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb814d9000, cur 1585024115 expire 1585023965 last 1585023888 [460693.520223] Lustre: Skipped 5 previous similar messages [460704.705990] Lustre: fir-OST0030: Connection restored to 2a9a2350-c0db-4 (at 10.49.29.4@o2ib1) [460704.714600] Lustre: Skipped 5 previous similar messages [460806.583604] Lustre: fir-OST0032: Connection restored to 8db8300c-7159-4 (at 10.50.14.14@o2ib2) [460806.592307] Lustre: Skipped 5 previous similar messages [462000.790455] LustreError: 96830:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 00d873ba-49ea-4 claims 1605632 GRANT, real grant 229376 [462895.668898] Lustre: fir-OST0030: Connection restored to efbc6add-d7ba-4 (at 10.50.12.4@o2ib2) [462895.677582] Lustre: Skipped 5 previous similar messages [464249.491304] Lustre: fir-OST0030: haven't heard from client 17ee7eca-4216-4 (at 10.50.13.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80eb7c00, cur 1585027671 expire 1585027521 last 1585027444 [464249.512318] Lustre: Skipped 5 previous similar messages [465558.801217] Lustre: fir-OST0030: Connection restored to 17ee7eca-4216-4 (at 10.50.13.5@o2ib2) [465558.809849] Lustre: Skipped 2 previous similar messages [465689.486665] Lustre: fir-OST0030: haven't heard from client f7fe261e-a413-4 (at 10.49.28.2@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80f05800, cur 1585029111 expire 1585028961 last 1585028884 [465689.506751] Lustre: Skipped 5 previous similar messages [465823.231839] Lustre: fir-OST0030: Connection restored to d2f2b8c6-0661-4 (at 10.50.16.5@o2ib2) [465823.240523] Lustre: Skipped 5 previous similar messages [469661.806472] Lustre: fir-OST0030: Connection restored to 6c45c03c-4b15-4 (at 10.50.16.6@o2ib2) [469661.815088] Lustre: Skipped 5 previous similar messages [470356.477005] Lustre: fir-OST0038: haven't heard from client c4422e40-3cfa-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfe456c00, cur 1585033778 expire 1585033628 last 1585033551 [470356.496977] Lustre: Skipped 11 previous similar messages [470400.758901] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [470400.767517] Lustre: Skipped 5 previous similar messages [473247.151351] LustreError: 48940:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 653b182b-aad1-4 claims 16752640 GRANT, real grant 10579968 [473273.869896] LustreError: 49388:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 1757184 GRANT, real grant 741376 [473459.472432] Lustre: fir-OST0036: haven't heard from client 6e31591f-913a-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80e000, cur 1585036881 expire 1585036731 last 1585036654 [473459.492415] Lustre: Skipped 5 previous similar messages [473482.480616] Lustre: fir-OST0034: haven't heard from client 6e31591f-913a-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd932b29000, cur 1585036904 expire 1585036754 last 1585036677 [473482.500587] Lustre: Skipped 4 previous similar messages [473539.499538] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [473539.508158] Lustre: Skipped 5 previous similar messages [475132.619882] Lustre: fir-OST0030: Connection restored to 33701c5f-e220-4 (at 10.50.13.12@o2ib2) [475132.628588] Lustre: Skipped 4 previous similar messages [475146.477673] Lustre: fir-OST0032: haven't heard from client bd12e5c8-ece9-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffca9b3bc00, cur 1585038568 expire 1585038418 last 1585038341 [475150.470278] Lustre: fir-OST0030: haven't heard from client bd12e5c8-ece9-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fda44379400, cur 1585038572 expire 1585038422 last 1585038345 [475150.490254] Lustre: Skipped 3 previous similar messages [475393.743774] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [475393.752394] Lustre: Skipped 6 previous similar messages [478003.381214] LustreError: 49234:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli f3287512-5093-4 claims 16752640 GRANT, real grant 8404992 [478043.616166] LustreError: 48326:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 9891086f-7733-4 claims 16752640 GRANT, real grant 4673536 [480362.457946] Lustre: fir-OST0034: haven't heard from client 0fe03f2f-af46-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00beeab8c00, cur 1585043784 expire 1585043634 last 1585043557 [480362.477949] Lustre: Skipped 1 previous similar message [480369.486838] Lustre: fir-OST0030: haven't heard from client 0fe03f2f-af46-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbef4ce000, cur 1585043791 expire 1585043641 last 1585043564 [480369.506870] Lustre: Skipped 1 previous similar message [480370.490676] Lustre: fir-OST0032: haven't heard from client 0fe03f2f-af46-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf7fee000, cur 1585043792 expire 1585043642 last 1585043565 [480370.510708] Lustre: Skipped 1 previous similar message [480376.461803] Lustre: fir-OST003a: haven't heard from client 0fe03f2f-af46-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bce6d2400, cur 1585043798 expire 1585043648 last 1585043571 [480424.284623] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [480424.293241] Lustre: Skipped 4 previous similar messages [484844.290084] LustreError: 49364:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli fa1e1112-266d-4 claims 28672 GRANT, real grant 16384 [489192.382462] LustreError: 49357:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 75520abd-92af-4 claims 1441792 GRANT, real grant 0 [489657.823637] LustreError: 49016:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli c2ca4c5a-e67e-4 claims 4169728 GRANT, real grant 2519040 [492417.437115] Lustre: fir-OST0038: haven't heard from client ee56fbe2-040d-4 (at 10.49.25.17@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb927fd000, cur 1585055839 expire 1585055689 last 1585055612 [493800.954409] LustreError: 49021:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli c2ca4c5a-e67e-4 claims 159744 GRANT, real grant 155648 [493898.459609] Lustre: fir-OST0036: haven't heard from client d8c1b053-4ffb-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf5323000, cur 1585057320 expire 1585057170 last 1585057093 [493898.479629] Lustre: Skipped 5 previous similar messages [493915.435502] Lustre: fir-OST003a: haven't heard from client d8c1b053-4ffb-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e0c800, cur 1585057337 expire 1585057187 last 1585057110 [493915.455501] Lustre: Skipped 4 previous similar messages [493983.009715] Lustre: fir-OST0032: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [493983.018335] Lustre: Skipped 6 previous similar messages [500669.445851] LustreError: 56059:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a67a88ce-e5ae-4 claims 16728064 GRANT, real grant 12271616 [501065.602234] LustreError: 49350:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli d0ed2bdc-7329-4 claims 376832 GRANT, real grant 192512 [501937.420097] Lustre: fir-OST0032: haven't heard from client 89912c83-1f62-4 (at 10.50.14.5@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7dc96800, cur 1585065359 expire 1585065209 last 1585065132 [503880.938323] Lustre: fir-OST0030: Connection restored to 89912c83-1f62-4 (at 10.50.14.5@o2ib2) [503880.946940] Lustre: Skipped 5 previous similar messages [504024.662203] Lustre: 47417:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585067438/real 1585067438] req@ffffa00bf1d2ad00 x1661537383144448/t0(0) o400->MGC10.0.10.51@o2ib7@10.0.10.51@o2ib7:26/25 lens 224/224 e 0 to 1 dl 1585067445 ref 1 fl Rpc:XN/0/ffffffff rc 0/-1 [504024.690378] Lustre: 47417:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 8 previous similar messages [504024.700236] LustreError: 166-1: MGC10.0.10.51@o2ib7: Connection to MGS (at 10.0.10.51@o2ib7) was lost; in progress operations using this service will fail [504030.312226] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Timed out tx for 10.0.10.51@o2ib7: 0 seconds [504030.322399] LNet: 47354:0:(o2iblnd_cb.c:3397:kiblnd_check_conns()) Skipped 14 previous similar messages [504030.331895] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) ni 10.0.10.109@o2ib7 added to recovery queue. Health = 900 [504030.343988] LNetError: 47354:0:(lib-msg.c:485:lnet_handle_local_failure()) Skipped 1 previous similar message [504051.673229] Lustre: fir-MDT0000-lwp-OST003a: Connection to fir-MDT0000 (at 10.0.10.52@o2ib7) was lost; in progress operations using this service will wait for recovery to complete [504051.687114] LustreError: 11-0: fir-MDT0000-lwp-OST003a: operation ldlm_enqueue to node 10.0.10.52@o2ib7 failed: rc = -107 [504051.687121] LustreError: Skipped 38 previous similar messages [504051.706168] Lustre: Skipped 5 previous similar messages [504229.110608] Lustre: fir-OST0030: Connection restored to fir-MDT0000-mdtlov_UUID (at 10.0.10.51@o2ib7) [504229.119921] Lustre: Skipped 3 previous similar messages [504275.557742] LustreError: 167-0: fir-MDT0000-lwp-OST0030: This client was evicted by fir-MDT0000; in progress operations using this service will fail. [504275.571210] LustreError: Skipped 5 previous similar messages [504275.578427] LustreError: 47401:0:(client.c:1197:ptlrpc_import_delay_req()) @@@ invalidate in flight req@ffff9ff4e2bc2400 x1661537384664000/t0(0) o103->fir-MDT0000-lwp-OST0036@10.0.10.51@o2ib7:17/18 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:W/0/ffffffff rc 0/-1 [504275.606920] Lustre: fir-MDT0000-lwp-OST0034: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [504275.616654] Lustre: Skipped 5 previous similar messages [504275.782026] LustreError: 11-0: fir-MDT0000-lwp-OST0032: operation quota_acquire to node 10.0.10.51@o2ib7 failed: rc = -11 [504275.793141] LustreError: Skipped 21 previous similar messages [504310.883488] LustreError: 11-0: fir-MDT0000-lwp-OST0030: operation quota_acquire to node 10.0.10.51@o2ib7 failed: rc = -11 [504310.894594] LustreError: Skipped 150 previous similar messages [504320.760704] Lustre: fir-OST0036: deleting orphan objects from 0x0:91599508 to 0x0:91600193 [504320.760705] Lustre: fir-OST0034: deleting orphan objects from 0x0:92729081 to 0x0:92729377 [504320.760706] Lustre: fir-OST0032: deleting orphan objects from 0x0:90672198 to 0x0:90672737 [504320.760754] Lustre: fir-OST0030: deleting orphan objects from 0x0:91256287 to 0x0:91257121 [504320.760827] Lustre: fir-OST0038: deleting orphan objects from 0x0:93364164 to 0x0:93364833 [504320.760885] Lustre: fir-OST003a: deleting orphan objects from 0x0:93272990 to 0x0:93273697 [504325.733669] Lustre: Evicted from MGS (at 10.0.10.51@o2ib7) after server handle changed from 0xae6d2c631bd48d67 to 0x764984320c7ae5f7 [504325.745816] Lustre: MGC10.0.10.51@o2ib7: Connection restored to 10.0.10.51@o2ib7 (at 10.0.10.51@o2ib7) [504398.722256] Lustre: fir-OST0030: Connection restored to f7fe261e-a413-4 (at 10.49.28.2@o2ib1) [504398.730879] Lustre: Skipped 5 previous similar messages [504593.417494] Lustre: fir-OST003a: haven't heard from client 08262875-245e-4 (at 10.50.15.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7eed6400, cur 1585068015 expire 1585067865 last 1585067788 [504593.437586] Lustre: Skipped 5 previous similar messages [505803.612750] Lustre: fir-OST0030: Connection restored to 08262875-245e-4 (at 10.50.15.13@o2ib2) [505803.621456] Lustre: Skipped 16 previous similar messages [507032.005234] Lustre: fir-OST0030: Connection restored to ce67c0db-bb4c-4 (at 10.50.15.5@o2ib2) [507032.013850] Lustre: Skipped 5 previous similar messages [507076.605864] Lustre: fir-OST0030: Connection restored to 6f426f61-5639-4 (at 10.50.13.10@o2ib2) [507076.614566] Lustre: Skipped 5 previous similar messages [508893.406317] Lustre: fir-OST003a: haven't heard from client c4e4a9fa-0416-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e0c000, cur 1585072315 expire 1585072165 last 1585072088 [508893.426319] Lustre: Skipped 5 previous similar messages [508955.240443] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [508955.249055] Lustre: Skipped 5 previous similar messages [510009.404356] Lustre: fir-OST0038: haven't heard from client 82c5fb51-6200-4 (at 10.50.16.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb92592800, cur 1585073431 expire 1585073281 last 1585073204 [510009.424416] Lustre: Skipped 5 previous similar messages [511421.284514] LustreError: 48341:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 5682229c-f010-4 claims 376832 GRANT, real grant 188416 [511752.792886] Lustre: fir-OST0032: Connection restored to 82c5fb51-6200-4 (at 10.50.16.9@o2ib2) [511752.801498] Lustre: Skipped 5 previous similar messages [512242.114974] LustreError: 49342:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 4169728 GRANT, real grant 2342912 [513395.396686] Lustre: fir-OST0038: haven't heard from client d4aafd71-b333-4 (at 10.50.14.15@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfeaf4400, cur 1585076817 expire 1585076667 last 1585076590 [513395.416758] Lustre: Skipped 5 previous similar messages [514567.669148] Lustre: fir-OST0030: Connection restored to d4aafd71-b333-4 (at 10.50.14.15@o2ib2) [514567.677853] Lustre: Skipped 4 previous similar messages [514812.901649] LustreError: 49347:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli bc3a5e42-5951-4 claims 16752640 GRANT, real grant 9310208 [516243.662043] LustreError: 49202:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 5616204e-fc92-4 claims 16752640 GRANT, real grant 5431296 [519132.573055] LustreError: 49342:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli e5696f4f-26e9-4 claims 28672 GRANT, real grant 0 [519219.384140] Lustre: fir-OST0032: haven't heard from client abed66c3-995c-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931932800, cur 1585082641 expire 1585082491 last 1585082414 [519219.404825] Lustre: Skipped 5 previous similar messages [519250.034935] Lustre: fir-OST0032: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [519250.043553] Lustre: Skipped 6 previous similar messages [519466.384371] Lustre: fir-OST0034: haven't heard from client 9f02a5e3-9696-4 (at 10.50.13.13@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ff38000, cur 1585082888 expire 1585082738 last 1585082661 [519466.404437] Lustre: Skipped 5 previous similar messages [521397.405259] Lustre: fir-OST0030: Connection restored to 9f02a5e3-9696-4 (at 10.50.13.13@o2ib2) [521397.413957] Lustre: Skipped 5 previous similar messages [522336.267477] LustreError: 49239:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 16703488 GRANT, real grant 3944448 [522829.377123] Lustre: fir-OST0030: haven't heard from client 72866633-325f-4 (at 10.50.15.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80018000, cur 1585086251 expire 1585086101 last 1585086024 [522829.397141] Lustre: Skipped 5 previous similar messages [523120.997837] LustreError: 49237:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli e5696f4f-26e9-4 claims 32768 GRANT, real grant 28672 [524540.277523] Lustre: fir-OST0030: Connection restored to 72866633-325f-4 (at 10.50.15.9@o2ib2) [524540.286186] Lustre: Skipped 3 previous similar messages [526776.499906] LustreError: 49334:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 732c4259-5045-4 claims 9945088 GRANT, real grant 6176768 [527520.368594] Lustre: fir-OST0036: haven't heard from client e9949bf5-ceeb-4 (at 10.50.12.12@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d6b7800, cur 1585090942 expire 1585090792 last 1585090715 [527520.388673] Lustre: Skipped 5 previous similar messages [528317.792689] LustreError: 49310:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a61ba52f-0e9d-4 claims 2060288 GRANT, real grant 647168 [528647.159299] LustreError: 49014:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 27b32a6c-5b01-4 claims 28672 GRANT, real grant 0 [528652.375803] Lustre: fir-OST0030: Connection restored to e9949bf5-ceeb-4 (at 10.50.12.12@o2ib2) [528652.384506] Lustre: Skipped 6 previous similar messages [533372.394382] LustreError: 49148:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli ee09de85-2d60-4 claims 16752640 GRANT, real grant 16023552 [533818.631787] Lustre: fir-OST0030: Connection restored to da59e233-110f-4 (at 10.50.13.15@o2ib2) [533818.640490] Lustre: Skipped 5 previous similar messages [534484.526029] LustreError: 49343:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 75daa1e6-b38d-4 claims 8441856 GRANT, real grant 196608 [537897.132107] LustreError: 48341:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a1b5a428-453b-4 claims 28672 GRANT, real grant 20480 [539093.278783] LustreError: 48896:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a61ba52f-0e9d-4 claims 16752640 GRANT, real grant 2084864 [540362.929602] LustreError: 102171:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 75daa1e6-b38d-4 claims 16654336 GRANT, real grant 8368128 [547250.650797] LustreError: 49282:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli c2ca4c5a-e67e-4 claims 16752640 GRANT, real grant 3162112 [547828.531504] LustreError: 49232:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 28672 GRANT, real grant 8192 [547921.446878] LustreError: 49150:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 653b182b-aad1-4 claims 28672 GRANT, real grant 24576 [548922.322097] Lustre: fir-OST0030: haven't heard from client d95d81db-621a-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd050bae800, cur 1585112344 expire 1585112194 last 1585112117 [548922.342117] Lustre: Skipped 5 previous similar messages [548970.850745] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [548970.859382] Lustre: Skipped 5 previous similar messages [549198.974474] LustreError: 48964:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a67a88ce-e5ae-4 claims 28672 GRANT, real grant 20480 [549578.834568] Lustre: fir-OST0030: Connection restored to 88c27327-e888-4 (at 10.49.27.21@o2ib1) [549578.843354] Lustre: Skipped 5 previous similar messages [553109.768510] LustreError: 49015:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 0271c837-bb5b-4 claims 1654784 GRANT, real grant 1486848 [555509.646247] LustreError: 49016:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 86db2c50-4e1f-4 claims 16752640 GRANT, real grant 888832 [558857.303452] Lustre: fir-OST0038: haven't heard from client 9cf8861f-40bc-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef16800, cur 1585122279 expire 1585122129 last 1585122052 [558857.323450] Lustre: Skipped 5 previous similar messages [558904.865490] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [558904.865491] Lustre: fir-OST0032: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [558904.882705] Lustre: Skipped 4 previous similar messages [561139.299443] Lustre: fir-OST0038: haven't heard from client 140a8048-bada-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930321c00, cur 1585124561 expire 1585124411 last 1585124334 [561139.319455] Lustre: Skipped 5 previous similar messages [561218.643734] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [561218.652349] Lustre: Skipped 4 previous similar messages [562725.298107] Lustre: fir-OST003a: haven't heard from client 69ede8ab-5c22-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf72b3800, cur 1585126147 expire 1585125997 last 1585125920 [562725.318080] Lustre: Skipped 5 previous similar messages [562765.781387] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [562765.790003] Lustre: Skipped 5 previous similar messages [568376.105382] LustreError: 49351:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3da00d75-6c3b-4 claims 344064 GRANT, real grant 139264 [569187.934100] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132602/real 1585132602] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132609 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [569194.961090] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132609/real 1585132609] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132616 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569201.988076] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132616/real 1585132616] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132623 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569209.015062] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132623/real 1585132623] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132630 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569216.042052] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132630/real 1585132630] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132637 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569230.069033] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132644/real 1585132644] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132651 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569230.096542] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 1 previous similar message [569251.105993] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132665/real 1585132665] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132672 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569251.133379] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [569286.142928] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132700/real 1585132700] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132707 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569286.170325] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [569356.179812] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585132770/real 1585132770] req@ffffa00bfa741680 x1661537613811200/t0(0) o106->fir-OST003a@10.50.6.54@o2ib2:15/16 lens 296/280 e 0 to 1 dl 1585132777 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [569356.207174] Lustre: 48966:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 9 previous similar messages [569381.474762] LNet: Service thread pid 48966 was inactive for 200.54s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [569381.491828] Pid: 48966, comm: ll_ost02_039 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [569381.502395] Call Trace: [569381.504977] [] ptlrpc_set_wait+0x480/0x790 [ptlrpc] [569381.511735] [] ldlm_run_ast_work+0xd5/0x3a0 [ptlrpc] [569381.518802] [] ldlm_glimpse_locks+0x3b/0x100 [ptlrpc] [569381.525714] [] ofd_intent_policy+0x69b/0x920 [ofd] [569381.532328] [] ldlm_lock_enqueue+0x356/0xa20 [ptlrpc] [569381.539193] [] ldlm_handle_enqueue0+0xa56/0x15f0 [ptlrpc] [569381.546406] [] tgt_enqueue+0x62/0x210 [ptlrpc] [569381.552672] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [569381.559771] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [569381.567643] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [569381.574100] [] kthread+0xd1/0xe0 [569381.579140] [] ret_from_fork_nospec_begin+0xe/0x21 [569381.585715] [] 0xffffffffffffffff [569381.590920] LustreError: dumping log to /tmp/lustre-log.1585132803.48966 [569390.295770] Lustre: fir-OST0032: haven't heard from client 53fb438a-2a45-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00a38a54000, cur 1585132812 expire 1585132662 last 1585132585 [569390.315787] Lustre: Skipped 5 previous similar messages [569403.289407] Lustre: fir-OST003a: haven't heard from client 53fb438a-2a45-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd3a2d800, cur 1585132825 expire 1585132675 last 1585132598 [569403.309402] Lustre: Skipped 4 previous similar messages [569403.314776] LustreError: 48966:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.50.6.54@o2ib2) failed to reply to glimpse AST (req@ffffa00bfa741680 x1661537613811200 status 0 rc -5), evict it ns: filter-fir-OST003a_UUID lock: ffff9fcc26d64800/0xf3793efb12209ea8 lrc: 3/0,0 mode: PW/PW res: [0x1180000402:0xdc4584:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 34359738368->18446744073709551615) flags: 0x40000000020000 nid: 10.50.6.54@o2ib2 remote: 0x31a77bbf4e9ba007 expref: 6 pid: 48860 timeout: 0 lvb_type: 0 [569403.361849] LustreError: 138-a: fir-OST003a: A client on nid 10.50.6.54@o2ib2 was evicted due to a lock glimpse callback time out: rc -5 [569403.374208] LustreError: Skipped 7 previous similar messages [569403.380007] LNet: Service thread pid 48966 completed after 222.44s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). [569446.809468] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [569446.818103] Lustre: Skipped 3 previous similar messages [571845.265762] LustreError: 49197:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli e5696f4f-26e9-4 claims 143360 GRANT, real grant 32768 [572734.279275] Lustre: fir-OST003a: haven't heard from client 568515af-b94a-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e09c00, cur 1585136156 expire 1585136006 last 1585135929 [572786.347241] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [572786.355858] Lustre: Skipped 6 previous similar messages [575631.277155] Lustre: fir-OST003a: haven't heard from client 51e206d5-3327-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9616c00, cur 1585139053 expire 1585138903 last 1585138826 [575631.297154] Lustre: Skipped 5 previous similar messages [575646.297666] Lustre: fir-OST0036: haven't heard from client 51e206d5-3327-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbeae4f800, cur 1585139068 expire 1585138918 last 1585138841 [575717.827935] Lustre: fir-OST0032: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [575717.836552] Lustre: Skipped 4 previous similar messages [579181.270527] Lustre: fir-OST0036: haven't heard from client e685b13a-a4f9-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5e70c00, cur 1585142603 expire 1585142453 last 1585142376 [579181.290521] Lustre: Skipped 4 previous similar messages [579185.269168] Lustre: fir-OST003a: haven't heard from client e685b13a-a4f9-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf902c800, cur 1585142607 expire 1585142457 last 1585142380 [579185.289164] Lustre: Skipped 4 previous similar messages [579232.022014] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [579232.030630] Lustre: Skipped 5 previous similar messages [579930.267426] Lustre: fir-OST0036: haven't heard from client f32d3637-39fe-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd1f1504400, cur 1585143352 expire 1585143202 last 1585143125 [579972.686306] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [579972.694926] Lustre: Skipped 5 previous similar messages [585443.260683] Lustre: fir-OST0030: haven't heard from client dadffa87-4553-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b83b48400, cur 1585148865 expire 1585148715 last 1585148638 [585443.280726] Lustre: Skipped 5 previous similar messages [585498.220217] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [585498.228837] Lustre: Skipped 5 previous similar messages [587180.267169] Lustre: fir-OST0038: haven't heard from client 2d5a47ed-4b4c-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb91dbdc00, cur 1585150602 expire 1585150452 last 1585150375 [587180.287141] Lustre: Skipped 5 previous similar messages [587230.238350] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [587230.246966] Lustre: Skipped 5 previous similar messages [592737.737160] LustreError: 49076:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 1f9d385a-f33f-4 claims 16752640 GRANT, real grant 376832 [592969.473812] LustreError: 49348:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 665344bc-9abf-4 claims 319488 GRANT, real grant 192512 [595594.557048] Lustre: fir-OST0030: Connection restored to 51620f49-4677-4 (at 10.50.15.2@o2ib2) [595594.565661] Lustre: Skipped 4 previous similar messages [600184.127789] LustreError: 49021:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli cda196c0-f220-4 claims 536576 GRANT, real grant 0 [604685.855337] LustreError: 67480:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli b4929648-8e2a-4 claims 28672 GRANT, real grant 12288 [604856.314350] LustreError: 67484:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 87383dee-e650-4 claims 28672 GRANT, real grant 24576 [605348.293204] LustreError: 49245:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 7940c6f6-66d7-4 claims 6000640 GRANT, real grant 5382144 [605402.381769] LustreError: 48892:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli bdf290df-3ae9-4 claims 28672 GRANT, real grant 20480 [605492.584365] LustreError: 49250:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 87383dee-e650-4 claims 28672 GRANT, real grant 24576 [605568.656488] LustreError: 48892:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 7940c6f6-66d7-4 claims 28672 GRANT, real grant 4096 [606698.214704] Lustre: fir-OST0030: haven't heard from client d7618323-c1e8-4 (at 10.50.14.11@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80fca800, cur 1585170120 expire 1585169970 last 1585169893 [606698.234797] Lustre: Skipped 5 previous similar messages [607572.282869] LustreError: 48932:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli b222a878-96be-4 claims 28672 GRANT, real grant 20480 [608605.148057] Lustre: fir-OST0030: Connection restored to d7618323-c1e8-4 (at 10.50.14.11@o2ib2) [608605.156762] Lustre: Skipped 6 previous similar messages [608955.401641] LustreError: 49245:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli f3287512-5093-4 claims 28672 GRANT, real grant 4096 [609150.136589] LustreError: 49323:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 28672 GRANT, real grant 20480 [609157.481745] LustreError: 49312:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 4a774151-5198-4 claims 28672 GRANT, real grant 20480 [609733.162052] LustreError: 49217:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6deab853-679e-4 claims 28672 GRANT, real grant 24576 [610327.631696] LustreError: 49139:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 28672 GRANT, real grant 16384 [621791.915644] LustreError: 49255:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a9a870dd-42ae-4 claims 28672 GRANT, real grant 4096 [624743.175251] Lustre: fir-OST0034: haven't heard from client 68edeab9-3169-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93773a000, cur 1585188165 expire 1585188015 last 1585187938 [624743.195307] Lustre: Skipped 5 previous similar messages [624749.222522] Lustre: fir-OST003a: haven't heard from client 68edeab9-3169-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b5b000, cur 1585188171 expire 1585188021 last 1585187944 [624749.242613] Lustre: Skipped 4 previous similar messages [624787.644490] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [624787.653107] Lustre: Skipped 5 previous similar messages [625909.909971] LustreError: 49327:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3da00d75-6c3b-4 claims 2895872 GRANT, real grant 344064 [626594.170579] Lustre: fir-OST0038: haven't heard from client 1015a07c-5c4c-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd1f1501800, cur 1585190016 expire 1585189866 last 1585189789 [626656.677050] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [626656.685666] Lustre: Skipped 4 previous similar messages [628489.165878] Lustre: fir-OST0036: haven't heard from client af897907-9595-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf7e0a800, cur 1585191911 expire 1585191761 last 1585191684 [628489.186088] Lustre: Skipped 5 previous similar messages [628492.165853] Lustre: fir-OST003a: haven't heard from client af897907-9595-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00beeab9400, cur 1585191914 expire 1585191764 last 1585191687 [628492.185995] Lustre: Skipped 4 previous similar messages [628529.900998] Lustre: fir-OST0032: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [628529.900999] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [628529.918258] Lustre: Skipped 4 previous similar messages [628627.057793] LustreError: 49310:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 665344bc-9abf-4 claims 15376384 GRANT, real grant 540672 [635251.462540] LustreError: 67480:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 14295040 GRANT, real grant 12242944 [642360.603976] LustreError: 49128:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 14688256 GRANT, real grant 14295040 [643150.084563] LustreError: 49245:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 665344bc-9abf-4 claims 16752640 GRANT, real grant 15376384 [644959.137263] LustreError: 49265:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dd1fe72f-e583-4 claims 16752640 GRANT, real grant 4730880 [645253.078688] LustreError: 49336:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6dd0a0db-fe9d-4 claims 4169728 GRANT, real grant 2973696 [647565.117630] perf: interrupt took too long (3930 > 3915), lowering kernel.perf_event_max_sample_rate to 50000 [673338.832739] LustreError: 49259:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 15421440 GRANT, real grant 14688256 [679269.868554] LustreError: 49238:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 6479814a-e93b-4 claims 450560 GRANT, real grant 57344 [679531.610551] Lustre: fir-OST0030: Connection restored to 38135a83-ec5c-4 (at 10.50.12.5@o2ib2) [679531.619163] Lustre: Skipped 4 previous similar messages [695096.051693] Lustre: fir-OST003a: haven't heard from client 6dcf0e67-1d32-4 (at 10.50.7.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb80895000, cur 1585258518 expire 1585258368 last 1585258291 [695106.069880] Lustre: fir-OST0036: haven't heard from client 6dcf0e67-1d32-4 (at 10.50.7.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7cc4f800, cur 1585258528 expire 1585258378 last 1585258301 [695109.049963] Lustre: fir-OST0030: haven't heard from client 6dcf0e67-1d32-4 (at 10.50.7.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7f460000, cur 1585258531 expire 1585258381 last 1585258304 [695109.754689] Lustre: fir-OST0032: Connection restored to 6dcf0e67-1d32-4 (at 10.50.7.9@o2ib2) [695109.754689] Lustre: fir-OST0030: Connection restored to 6dcf0e67-1d32-4 (at 10.50.7.9@o2ib2) [695109.754692] Lustre: Skipped 1 previous similar message [695109.777018] Lustre: Skipped 4 previous similar messages [695116.043420] Lustre: fir-OST0034: haven't heard from client 6dcf0e67-1d32-4 (at 10.50.7.9@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ff66800, cur 1585258538 expire 1585258388 last 1585258311 [695116.063402] Lustre: Skipped 1 previous similar message [695403.633238] LustreError: 49331:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 7822258b-c265-4 claims 16752640 GRANT, real grant 28672 [696277.177905] Lustre: fir-OST0030: Connection restored to 35cd2c2f-e02c-4 (at 10.50.12.13@o2ib2) [696277.186637] Lustre: Skipped 5 previous similar messages [698289.301487] LustreError: 49388:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli dff4fc43-431c-4 claims 28672 GRANT, real grant 12288 [711065.506092] LustreError: 49273:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli d0ed2bdc-7329-4 claims 1093632 GRANT, real grant 376832 [719937.473576] LustreError: 49388:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 00d873ba-49ea-4 claims 376832 GRANT, real grant 303104 [721450.977406] LustreError: 48944:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 02131258-2b02-4 claims 16752640 GRANT, real grant 8118272 [721973.869404] LustreError: 49016:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 16109568 GRANT, real grant 15421440 [734574.428932] LustreError: 102174:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 2147e45e-5aee-4 claims 28672 GRANT, real grant 4096 [735252.934320] LustreError: 102175:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 39b158e5-2e75-4 claims 28672 GRANT, real grant 20480 [736169.131557] LustreError: 49075:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 88de242f-0ae5-4 claims 28672 GRANT, real grant 20480 [736751.997724] LustreError: 49333:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 88de242f-0ae5-4 claims 28672 GRANT, real grant 12288 [738163.868582] LustreError: 49309:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 732c4259-5045-4 claims 16752640 GRANT, real grant 9945088 [738386.483078] LustreError: 49380:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 16187392 GRANT, real grant 16109568 [746718.607191] LustreError: 49277:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 9342fda8-7ae5-4 claims 16752640 GRANT, real grant 16211968 [748626.690560] LustreError: 49095:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 86db2c50-4e1f-4 claims 28672 GRANT, real grant 4096 [759516.435802] Lustre: fir-OST0030: Connection restored to cb22fe63-8523-4 (at 10.49.19.4@o2ib1) [759516.444444] Lustre: Skipped 5 previous similar messages [760351.975239] Lustre: fir-OST0036: haven't heard from client 3ab8a3cd-4496-4 (at 10.49.19.3@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d680800, cur 1585323774 expire 1585323624 last 1585323547 [760351.995245] Lustre: Skipped 1 previous similar message [761598.280768] Lustre: fir-OST0030: Connection restored to cb22fe63-8523-4 (at 10.49.19.4@o2ib1) [761598.289400] Lustre: Skipped 5 previous similar messages [761630.052470] Lustre: fir-OST0030: Connection restored to 3ab8a3cd-4496-4 (at 10.49.19.3@o2ib1) [761630.061093] Lustre: Skipped 5 previous similar messages [761798.593461] Lustre: fir-OST0030: Connection restored to 6eb34c30-3804-4 (at 10.49.19.6@o2ib1) [761798.602075] Lustre: Skipped 5 previous similar messages [762241.309734] Lustre: fir-OST0030: Connection restored to 3ab8a3cd-4496-4 (at 10.49.19.3@o2ib1) [762241.318353] Lustre: Skipped 5 previous similar messages [763389.176140] Lustre: fir-OST0030: Connection restored to 6f68bfc7-8d0d-4 (at 10.49.19.1@o2ib1) [763389.184777] Lustre: Skipped 11 previous similar messages [763416.936283] Lustre: fir-OST0030: Connection restored to 565e39cd-4898-4 (at 10.49.19.8@o2ib1) [763416.944902] Lustre: Skipped 5 previous similar messages [763453.008770] Lustre: fir-OST0030: Connection restored to 0b2cd2d4-168b-4 (at 10.49.19.7@o2ib1) [763453.017389] Lustre: Skipped 5 previous similar messages [763460.354049] Lustre: fir-OST0030: Connection restored to 9939e999-d857-4 (at 10.49.19.5@o2ib1) [763460.362663] Lustre: Skipped 5 previous similar messages [764403.597955] Lustre: fir-OST0030: Connection restored to 6eb34c30-3804-4 (at 10.49.19.6@o2ib1) [764403.606569] Lustre: Skipped 5 previous similar messages [764433.795881] Lustre: fir-OST0030: Connection restored to 3ab8a3cd-4496-4 (at 10.49.19.3@o2ib1) [764433.804500] Lustre: Skipped 11 previous similar messages [764453.214711] Lustre: fir-OST0030: Connection restored to 6f68bfc7-8d0d-4 (at 10.49.19.1@o2ib1) [764453.223363] Lustre: Skipped 5 previous similar messages [773013.730943] LustreError: 49299:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6dd0a0db-fe9d-4 claims 28672 GRANT, real grant 20480 [773375.168014] LustreError: 49367:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 322961a3-22bb-4 claims 28672 GRANT, real grant 8192 [775095.762478] Lustre: fir-OST0030: Connection restored to 0b2cd2d4-168b-4 (at 10.49.19.7@o2ib1) [775095.771093] Lustre: Skipped 5 previous similar messages [775102.872362] Lustre: fir-OST0030: Connection restored to 9939e999-d857-4 (at 10.49.19.5@o2ib1) [775102.880980] Lustre: Skipped 5 previous similar messages [775671.512387] LustreError: 48896:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 00d873ba-49ea-4 claims 94208 GRANT, real grant 28672 [776193.205562] LustreError: 49198:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 00d873ba-49ea-4 claims 15429632 GRANT, real grant 2560000 [777073.518885] LustreError: 49217:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli bdf290df-3ae9-4 claims 28672 GRANT, real grant 12288 [778975.780127] LustreError: 49286:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a09c1f00-b96c-4 claims 28672 GRANT, real grant 8192 [782129.735322] LustreError: 49083:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 75daa1e6-b38d-4 claims 28672 GRANT, real grant 20480 [783001.911151] Lustre: fir-OST0038: haven't heard from client 661e22b8-e9f5-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80f000, cur 1585346424 expire 1585346274 last 1585346197 [783001.931242] Lustre: Skipped 47 previous similar messages [783054.186906] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [783054.195529] Lustre: Skipped 3 previous similar messages [785922.901492] Lustre: fir-OST0030: haven't heard from client 144bfe18-3352-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb77947800, cur 1585349345 expire 1585349195 last 1585349118 [785922.921619] Lustre: Skipped 5 previous similar messages [785930.953665] Lustre: fir-OST0038: haven't heard from client 144bfe18-3352-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934fe800, cur 1585349353 expire 1585349203 last 1585349126 [785930.973675] Lustre: Skipped 1 previous similar message [785937.901738] Lustre: fir-OST0034: haven't heard from client 144bfe18-3352-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934fe000, cur 1585349360 expire 1585349210 last 1585349133 [785942.934350] Lustre: fir-OST0036: haven't heard from client 144bfe18-3352-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9ef0000, cur 1585349365 expire 1585349215 last 1585349138 [785942.954437] Lustre: Skipped 1 previous similar message [785985.205092] Lustre: fir-OST0032: Connection restored to 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) [785985.213710] Lustre: Skipped 5 previous similar messages [786707.782660] Lustre: fir-OST0030: Connection restored to f3e067b1-f2a3-4 (at 10.49.19.2@o2ib1) [786707.791296] Lustre: Skipped 3 previous similar messages [800320.583703] LustreError: 49011:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3da00d75-6c3b-4 claims 16752640 GRANT, real grant 2895872 [809278.385663] Lustre: fir-OST0030: Connection restored to 35cd2c2f-e02c-4 (at 10.50.12.13@o2ib2) [809278.394365] Lustre: Skipped 6 previous similar messages [844317.784946] Lustre: fir-OST0032: haven't heard from client a93d4107-2edc-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931935800, cur 1585407740 expire 1585407590 last 1585407513 [844418.731419] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [844418.740033] Lustre: Skipped 5 previous similar messages [861931.740802] Lustre: fir-OST0036: haven't heard from client 85b52166-baef-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd939e72800, cur 1585425354 expire 1585425204 last 1585425127 [861931.760819] Lustre: Skipped 5 previous similar messages [861976.630172] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [861976.638799] Lustre: Skipped 5 previous similar messages [872041.696297] Lustre: fir-OST0030: Connection restored to e71e196d-00dc-4 (at 10.50.5.33@o2ib2) [872041.704910] Lustre: Skipped 4 previous similar messages [872090.708560] Lustre: fir-OST003a: haven't heard from client 780d3450-c327-4 (at 10.50.5.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb5a2c00, cur 1585435513 expire 1585435363 last 1585435286 [872090.728633] Lustre: Skipped 5 previous similar messages [872092.710438] Lustre: fir-OST0032: haven't heard from client 780d3450-c327-4 (at 10.50.5.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbcd34a800, cur 1585435515 expire 1585435365 last 1585435288 [872092.730443] Lustre: Skipped 2 previous similar messages [872095.759553] Lustre: fir-OST0034: haven't heard from client 780d3450-c327-4 (at 10.50.5.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f5ae800, cur 1585435518 expire 1585435368 last 1585435291 [876328.601639] LNetError: 47371:0:(peer.c:3451:lnet_peer_ni_add_to_recoveryq_locked()) lpni 10.0.10.225@o2ib7 added to recovery queue. Health = 900 [876328.601651] LustreError: 49204:0:(ldlm_lib.c:3294:target_bulk_io()) @@@ truncated bulk READ 0(131072) req@ffff9feb7fda2850 x1662439050883712/t0(0) o3->add63273-c3b3-4@10.50.9.27@o2ib2:163/0 lens 488/440 e 1 to 0 dl 1585439763 ref 1 fl Interpret:/0/0 rc 0/0 [876328.601672] Lustre: fir-OST003a: Bulk IO read error with add63273-c3b3-4 (at 10.50.9.27@o2ib2), client will retry: rc -110 [876328.601673] Lustre: Skipped 3 previous similar messages [876478.695004] Lustre: fir-OST0034: haven't heard from client add63273-c3b3-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93011cc00, cur 1585439901 expire 1585439751 last 1585439674 [876478.715087] Lustre: Skipped 1 previous similar message [876496.693787] Lustre: fir-OST003a: haven't heard from client add63273-c3b3-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfe452c00, cur 1585439919 expire 1585439769 last 1585439692 [876496.713795] Lustre: Skipped 4 previous similar messages [876520.350572] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [876520.359191] Lustre: Skipped 6 previous similar messages [877467.313997] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585440882/real 1585440882] req@ffffa00bfa740900 x1661540417392704/t0(0) o104->fir-OST003a@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1585440889 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [877467.341333] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 6 previous similar messages [877488.350896] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585440903/real 1585440903] req@ffffa00bfa740900 x1661540417392704/t0(0) o104->fir-OST003a@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1585440910 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [877488.378237] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [877523.387791] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585440938/real 1585440938] req@ffffa00bfa740900 x1661540417392704/t0(0) o104->fir-OST003a@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1585440945 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [877523.415158] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [877593.425586] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1585441008/real 1585441008] req@ffffa00bfa740900 x1661540417392704/t0(0) o104->fir-OST003a@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1585441015 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [877593.452926] Lustre: 49037:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 9 previous similar messages [877614.462533] LustreError: 49037:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.50.9.27@o2ib2) failed to reply to blocking AST (req@ffffa00bfa740900 x1661540417392704 status 0 rc -110), evict it ns: filter-fir-OST003a_UUID lock: ffff9fd04ca4c380/0xf3793efb313de177 lrc: 4/0,0 mode: PW/PW res: [0x1180000401:0x81544d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) flags: 0x60000400030020 nid: 10.50.9.27@o2ib2 remote: 0xaa9511bccc19ced7 expref: 5 pid: 112664 timeout: 877758 lvb_type: 0 [877614.508180] LustreError: 138-a: fir-OST003a: A client on nid 10.50.9.27@o2ib2 was evicted due to a lock blocking callback time out: rc -110 [877614.520856] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 154s: evicting client at 10.50.9.27@o2ib2 ns: filter-fir-OST003a_UUID lock: ffff9fd04ca4c380/0xf3793efb313de177 lrc: 3/0,0 mode: PW/PW res: [0x1180000401:0x81544d:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) flags: 0x60000400030020 nid: 10.50.9.27@o2ib2 remote: 0xaa9511bccc19ced7 expref: 6 pid: 112664 timeout: 0 lvb_type: 0 [877614.561115] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) Skipped 4 previous similar messages [877679.708119] Lustre: fir-OST0032: haven't heard from client 9525333d-e5b4-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f570400, cur 1585441102 expire 1585440952 last 1585440875 [877681.696142] Lustre: fir-OST0036: haven't heard from client 9525333d-e5b4-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f577800, cur 1585441104 expire 1585440954 last 1585440877 [877681.716133] Lustre: Skipped 1 previous similar message [877716.902434] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [877716.911053] Lustre: Skipped 4 previous similar messages [878102.419239] LustreError: 48936:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 0271c837-bb5b-4 claims 4169728 GRANT, real grant 3059712 [896686.470349] LustreError: 49283:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2d94e235-5333-4 claims 28672 GRANT, real grant 0 [897107.643248] Lustre: fir-OST0036: haven't heard from client ecf003cb-b7be-4 (at 10.50.16.8@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbec5f9400, cur 1585460530 expire 1585460380 last 1585460303 [897107.663267] Lustre: Skipped 2 previous similar messages [898863.050810] Lustre: fir-OST0030: Connection restored to ecf003cb-b7be-4 (at 10.50.16.8@o2ib2) [898863.059428] Lustre: Skipped 6 previous similar messages [921925.121866] LustreError: 56064:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli dcb1fce2-5124-4 claims 28672 GRANT, real grant 20480 [929188.575981] Lustre: fir-OST0038: haven't heard from client bdc0155b-ddc0-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fcc02659000, cur 1585492611 expire 1585492461 last 1585492384 [929188.596005] Lustre: Skipped 5 previous similar messages [929232.444089] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [929232.452709] Lustre: Skipped 5 previous similar messages [929308.171672] LustreError: 49375:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6deab853-679e-4 claims 28672 GRANT, real grant 8192 [934702.558101] Lustre: fir-OST0036: haven't heard from client 144229db-845e-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5972400, cur 1585498125 expire 1585497975 last 1585497898 [934702.578143] Lustre: Skipped 5 previous similar messages [934750.847041] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [934750.855658] Lustre: Skipped 5 previous similar messages [936079.489436] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [936079.498072] Lustre: Skipped 5 previous similar messages [937279.247206] LustreError: 49363:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 4a774151-5198-4 claims 28672 GRANT, real grant 12288 [937612.555873] Lustre: fir-OST0034: haven't heard from client 6e79d69f-c682-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe3846e0000, cur 1585501035 expire 1585500885 last 1585500808 [937612.575926] Lustre: Skipped 5 previous similar messages [937615.556549] Lustre: fir-OST0032: haven't heard from client 6e79d69f-c682-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5aa000, cur 1585501038 expire 1585500888 last 1585500811 [937615.576574] Lustre: Skipped 3 previous similar messages [937650.579287] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [937650.587909] Lustre: Skipped 5 previous similar messages [938188.552741] Lustre: fir-OST0038: haven't heard from client dd1ee8c2-0420-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80bc00, cur 1585501611 expire 1585501461 last 1585501384 [938188.572846] Lustre: Skipped 1 previous similar message [938216.216689] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [938216.225312] Lustre: Skipped 5 previous similar messages [939731.785743] LustreError: 56060:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 75eb7c2d-56d1-4 claims 4141056 GRANT, real grant 0 [940530.904963] LustreError: 49337:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a1986753-a6b4-4 claims 28672 GRANT, real grant 8192 [942931.787644] LustreError: 49320:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 00d873ba-49ea-4 claims 28672 GRANT, real grant 8192 [945343.420922] LNetError: 47354:0:(o2iblnd_cb.c:3351:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds [945343.431176] LNetError: 47354:0:(o2iblnd_cb.c:3426:kiblnd_check_conns()) Timed out RDMA with 10.0.10.3@o2ib7 (106): c: 7, oc: 0, rc: 8 [945501.103260] Lustre: fir-OST0032: Connection restored to f82c1c78-5289-8d3f-c213-fe2a9908b1d9 (at 10.0.10.3@o2ib7) [945501.113620] Lustre: Skipped 4 previous similar messages [946558.520453] Lustre: fir-OST003a: haven't heard from client 0885c11b-bcdc-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb929000, cur 1585509981 expire 1585509831 last 1585509754 [946558.540436] Lustre: Skipped 5 previous similar messages [946616.576944] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [946616.585562] Lustre: Skipped 5 previous similar messages [949976.961486] LustreError: 48910:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 10cda2d8-aeb7-4 claims 3969024 GRANT, real grant 0 [955918.508306] Lustre: fir-OST003a: haven't heard from client 8c7a79ff-f053-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb9d98ac00, cur 1585519341 expire 1585519191 last 1585519114 [955918.528306] Lustre: Skipped 5 previous similar messages [955920.506799] Lustre: fir-OST0030: haven't heard from client 8c7a79ff-f053-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93773cc00, cur 1585519343 expire 1585519193 last 1585519116 [955922.495552] Lustre: fir-OST0032: haven't heard from client 8c7a79ff-f053-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdaad920800, cur 1585519345 expire 1585519195 last 1585519118 [955922.515521] Lustre: Skipped 3 previous similar messages [956012.041669] Lustre: fir-OST0032: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [956012.050283] Lustre: Skipped 6 previous similar messages [969586.810957] LustreError: 49219:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2d94e235-5333-4 claims 86016 GRANT, real grant 28672 [977552.827254] LustreError: 49257:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2d94e235-5333-4 claims 90112 GRANT, real grant 86016 [978835.453085] Lustre: fir-OST0038: haven't heard from client 141984c0-0bb7-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fcc048f4c00, cur 1585542258 expire 1585542108 last 1585542031 [978879.511605] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [978879.520221] Lustre: Skipped 5 previous similar messages [980561.453034] Lustre: fir-OST0034: haven't heard from client 3ca91764-a501-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930bc2000, cur 1585543984 expire 1585543834 last 1585543757 [980561.473021] Lustre: Skipped 5 previous similar messages [980609.503906] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [980609.512529] Lustre: Skipped 3 previous similar messages [983436.309625] LustreError: 49192:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli b73e1dab-2ad6-4 claims 2793472 GRANT, real grant 1687552 [985369.616550] LustreError: 49385:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 87383dee-e650-4 claims 28672 GRANT, real grant 16384 [988113.989416] LustreError: 49261:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli e1ffb0b6-0802-4 claims 28672 GRANT, real grant 0 [989039.429156] Lustre: fir-OST0030: haven't heard from client 76521fe8-386d-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd931932800, cur 1585552462 expire 1585552312 last 1585552235 [989039.449173] Lustre: Skipped 5 previous similar messages [989088.042472] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [989088.051084] Lustre: Skipped 5 previous similar messages [1008823.251602] LustreError: 48932:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli f8ebc65b-1d76-4 claims 28672 GRANT, real grant 0 [1020270.946138] Lustre: fir-OST0030: Connection restored to e71e196d-00dc-4 (at 10.50.5.33@o2ib2) [1020270.954846] Lustre: Skipped 5 previous similar messages [1020338.352156] Lustre: fir-OST003a: haven't heard from client fdd6ec41-5cf5-4 (at 10.50.5.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93854a400, cur 1585583761 expire 1585583611 last 1585583534 [1020338.372215] Lustre: Skipped 5 previous similar messages [1025799.370646] LustreError: 49246:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 5577ac59-ef38-4 claims 16752640 GRANT, real grant 0 [1029095.229472] LustreError: 48946:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 1f9d385a-f33f-4 claims 28672 GRANT, real grant 24576 [1032365.589098] LustreError: 49347:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli e1ffb0b6-0802-4 claims 1441792 GRANT, real grant 69632 [1035020.986836] LustreError: 48326:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli bc3a5e42-5951-4 claims 28672 GRANT, real grant 20480 [1035344.658355] LustreError: 49279:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 6dd0a0db-fe9d-4 claims 28672 GRANT, real grant 12288 [1036899.538321] LustreError: 49338:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2f954eed-4b6c-4 claims 28672 GRANT, real grant 20480 [1038246.980868] LustreError: 99543:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 95eea94a-c8bd-4 claims 28672 GRANT, real grant 4096 [1047532.300015] Lustre: fir-OST003a: haven't heard from client e47349d1-c447-4 (at 10.50.14.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd935e73800, cur 1585610955 expire 1585610805 last 1585610728 [1047532.320143] Lustre: Skipped 5 previous similar messages [1050123.684068] Lustre: fir-OST0030: Connection restored to 1c46010b-e86f-4 (at 10.50.14.3@o2ib2) [1050123.692776] Lustre: Skipped 3 previous similar messages [1050329.230719] Lustre: fir-OST0036: Client 0dc74af1-84ad-4 (at 10.49.0.61@o2ib1) reconnecting [1050329.239160] Lustre: Skipped 22 previous similar messages [1050329.244695] Lustre: fir-OST0036: Connection restored to 0dc74af1-84ad-4 (at 10.49.0.61@o2ib1) [1050329.784693] LustreError: 137-5: fir-OST0033_UUID: not available for connect from 10.49.0.61@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1050329.802212] LustreError: Skipped 9 previous similar messages [1050354.763627] LustreError: 137-5: fir-OST003b_UUID: not available for connect from 10.49.0.61@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1050800.294401] Lustre: fir-OST0038: haven't heard from client c0c00923-1be8-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf0eeec00, cur 1585614223 expire 1585614073 last 1585613996 [1050800.314495] Lustre: Skipped 5 previous similar messages [1050843.224058] Lustre: fir-OST0034: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1050843.232765] Lustre: Skipped 3 previous similar messages [1054687.765117] LustreError: 49127:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 5b88c4a2-39df-4 claims 28672 GRANT, real grant 12288 [1057265.603436] LustreError: 49306:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli f8ebc65b-1d76-4 claims 57344 GRANT, real grant 28672 [1072021.699416] LustreError: 49290:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli b13df3fd-f426-4 claims 1388544 GRANT, real grant 0 [1087120.010719] LustreError: 49332:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli e1ffb0b6-0802-4 claims 16670720 GRANT, real grant 1359872 [1094900.215400] Lustre: fir-OST0036: haven't heard from client e15458cc-5a0f-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0076752b000, cur 1585658323 expire 1585658173 last 1585658096 [1094900.235616] Lustre: Skipped 5 previous similar messages [1094945.714955] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1094945.723657] Lustre: Skipped 5 previous similar messages [1106030.320733] LustreError: 49360:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cbf4c94a-83d4-4 claims 28672 GRANT, real grant 12288 [1106979.559797] LustreError: 48908:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli cbf4c94a-83d4-4 claims 28672 GRANT, real grant 24576 [1107917.190315] Lustre: fir-OST0030: haven't heard from client 540b2ede-974e-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00862743800, cur 1585671340 expire 1585671190 last 1585671113 [1107917.210391] Lustre: Skipped 5 previous similar messages [1108006.071241] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1108006.079941] Lustre: Skipped 5 previous similar messages [1114551.872656] LustreError: 91666:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli b4929648-8e2a-4 claims 28672 GRANT, real grant 20480 [1118670.158163] Lustre: fir-OST0032: haven't heard from client 71c512fc-f1b5-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbba7a4800, cur 1585682093 expire 1585681943 last 1585681866 [1118670.178266] Lustre: Skipped 5 previous similar messages [1118720.702180] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1118720.710882] Lustre: Skipped 5 previous similar messages [1118879.334717] LustreError: 91670:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a1b5a428-453b-4 claims 28672 GRANT, real grant 16384 [1121365.701182] LustreError: 49277:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a1b5a428-453b-4 claims 28672 GRANT, real grant 24576 [1122218.901116] LustreError: 49369:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 81a86e52-614e-4 claims 512000 GRANT, real grant 0 [1123617.868631] LustreError: 49355:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli a09c1f00-b96c-4 claims 28672 GRANT, real grant 24576 [1125735.884346] LustreError: 49255:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 3a5b08eb-e519-4 claims 28672 GRANT, real grant 4096 [1126126.112583] LustreError: 49391:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli ee4b308c-5261-4 claims 28672 GRANT, real grant 4096 [1127467.027810] LustreError: 49280:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 95e66b92-7a58-4 claims 28672 GRANT, real grant 8192 [1128553.920716] LustreError: 49367:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli 5616204e-fc92-4 claims 28672 GRANT, real grant 16384 [1130356.299827] LustreError: 51674:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 39b158e5-2e75-4 claims 28672 GRANT, real grant 20480 [1130547.975482] LustreError: 49275:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 05dcd30f-3083-4 claims 733184 GRANT, real grant 0 [1132265.776391] LustreError: 91671:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 39b158e5-2e75-4 claims 28672 GRANT, real grant 24576 [1133468.997064] LustreError: 49290:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli a655f028-613f-4 claims 28672 GRANT, real grant 0 [1133678.100618] LustreError: 49242:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli a655f028-613f-4 claims 950272 GRANT, real grant 0 [1141087.116303] Lustre: fir-OST0032: haven't heard from client 21c8c585-5f1e-4 (at 10.50.8.18@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e3cec00, cur 1585704510 expire 1585704360 last 1585704283 [1141087.136405] Lustre: Skipped 5 previous similar messages [1170894.060129] Lustre: fir-OST0030: haven't heard from client 51a2b347-1573-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd050bae800, cur 1585734317 expire 1585734167 last 1585734090 [1170894.080225] Lustre: Skipped 5 previous similar messages [1170942.260173] Lustre: fir-OST0030: Connection restored to 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) [1170942.268871] Lustre: Skipped 5 previous similar messages [1192679.874601] LustreError: 49187:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli b13df3fd-f426-4 claims 28672 GRANT, real grant 0 [1197908.005070] Lustre: fir-OST0038: haven't heard from client 39fa7b7d-0126-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd49a26fc00, cur 1585761331 expire 1585761181 last 1585761104 [1197908.025144] Lustre: Skipped 5 previous similar messages [1197954.705677] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1197954.714376] Lustre: Skipped 5 previous similar messages [1203975.869481] Lustre: fir-OST0034: Client 40f84ba4-941c-4 (at 10.50.0.71@o2ib2) reconnecting [1203975.873002] Lustre: fir-OST0030: Connection restored to 40f84ba4-941c-4 (at 10.50.0.71@o2ib2) [1203975.886621] Lustre: Skipped 1 previous similar message [1204001.118176] LustreError: 137-5: fir-OST003b_UUID: not available for connect from 10.50.0.71@o2ib2 (no target). If you are running an HA pair check that the target is mounted on the other server. [1204001.135678] LustreError: Skipped 2 previous similar messages [1204026.205307] Lustre: fir-OST0034: Client 40f84ba4-941c-4 (at 10.50.0.71@o2ib2) reconnecting [1204026.213756] Lustre: Skipped 1 previous similar message [1204026.219155] Lustre: fir-OST0034: Connection restored to 40f84ba4-941c-4 (at 10.50.0.71@o2ib2) [1204026.227858] Lustre: Skipped 2 previous similar messages [1206360.390125] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1206360.398917] Lustre: Skipped 5 previous similar messages [1206405.987136] Lustre: fir-OST0034: haven't heard from client b449688d-0d11-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7fcc7c00, cur 1585769829 expire 1585769679 last 1585769602 [1206406.007358] Lustre: Skipped 5 previous similar messages [1206841.332613] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1206841.341439] Lustre: Skipped 5 previous similar messages [1206888.986825] Lustre: fir-OST0030: haven't heard from client f23e2d13-b3ae-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bce6d4400, cur 1585770312 expire 1585770162 last 1585770085 [1206889.007056] Lustre: Skipped 5 previous similar messages [1207444.987195] Lustre: fir-OST0030: haven't heard from client f0da1f04-da4c-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf77b1400, cur 1585770868 expire 1585770718 last 1585770641 [1207445.007391] Lustre: Skipped 5 previous similar messages [1207449.412218] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1207449.421012] Lustre: Skipped 5 previous similar messages [1222605.609804] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1222605.618593] Lustre: Skipped 5 previous similar messages [1222656.963428] Lustre: fir-OST0038: haven't heard from client 1b6de69e-1c49-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e4cc000, cur 1585786080 expire 1585785930 last 1585785853 [1222656.983566] Lustre: Skipped 5 previous similar messages [1244525.018047] LustreError: 91673:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 275b0d00-9e41-4 claims 1388544 GRANT, real grant 393216 [1248243.818737] LustreError: 49030:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 15e5b565-193d-4 claims 950272 GRANT, real grant 28672 [1249923.926186] LustreError: 49228:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 05dcd30f-3083-4 claims 950272 GRANT, real grant 733184 [1252701.919251] Lustre: fir-OST0030: haven't heard from client 479a23d1-e582-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f5800, cur 1585816125 expire 1585815975 last 1585815898 [1252701.939304] Lustre: Skipped 5 previous similar messages [1252769.659031] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1252769.667730] Lustre: Skipped 5 previous similar messages [1265760.202706] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1265760.211529] Lustre: Skipped 5 previous similar messages [1265808.903985] Lustre: fir-OST0032: haven't heard from client 7fa477f6-65a5-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf3245800, cur 1585829232 expire 1585829082 last 1585829005 [1265808.924158] Lustre: Skipped 5 previous similar messages [1266345.492402] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1266345.501219] Lustre: Skipped 4 previous similar messages [1266388.904722] Lustre: fir-OST0030: haven't heard from client b34989b7-841a-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbeada6000, cur 1585829812 expire 1585829662 last 1585829585 [1266388.924899] Lustre: Skipped 5 previous similar messages [1266597.903535] Lustre: fir-OST0034: haven't heard from client aaf569c5-b770-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbed037800, cur 1585830021 expire 1585829871 last 1585829794 [1266597.923712] Lustre: Skipped 5 previous similar messages [1266798.000889] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1266798.009676] Lustre: Skipped 5 previous similar messages [1267048.360798] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1267048.369632] Lustre: Skipped 5 previous similar messages [1267125.932472] Lustre: fir-OST0034: haven't heard from client 0bcab18e-ca9e-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf57f6400, cur 1585830549 expire 1585830399 last 1585830322 [1267125.952613] Lustre: Skipped 5 previous similar messages [1267442.296370] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1267442.305161] Lustre: Skipped 5 previous similar messages [1267476.903820] Lustre: fir-OST0032: haven't heard from client 23290f18-a3b6-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7b496400, cur 1585830900 expire 1585830750 last 1585830673 [1267476.923962] Lustre: Skipped 5 previous similar messages [1267945.903266] Lustre: fir-OST0030: haven't heard from client 4c28cfb2-f28b-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb814d9000, cur 1585831369 expire 1585831219 last 1585831142 [1267945.923405] Lustre: Skipped 5 previous similar messages [1270248.563606] Lustre: fir-OST0030: Connection restored to e71e196d-00dc-4 (at 10.50.5.33@o2ib2) [1270248.572307] Lustre: Skipped 5 previous similar messages [1270306.899018] Lustre: fir-OST0038: haven't heard from client df593509-e01a-4 (at 10.50.5.33@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ef47c00, cur 1585833730 expire 1585833580 last 1585833503 [1270306.919102] Lustre: Skipped 5 previous similar messages [1275215.837625] LustreError: 99543:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0030: cli cda196c0-f220-4 claims 16752640 GRANT, real grant 536576 [1275322.845683] LustreError: 49321:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli 3e41d262-577a-4 claims 28672 GRANT, real grant 0 [1275468.332339] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1275468.341129] Lustre: Skipped 5 previous similar messages [1276419.772862] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1276419.781747] Lustre: Skipped 5 previous similar messages [1276447.896279] Lustre: fir-OST0030: haven't heard from client 58a1fac0-9465-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb78d11c00, cur 1585839871 expire 1585839721 last 1585839644 [1276447.916420] Lustre: Skipped 5 previous similar messages [1276459.537957] Lustre: fir-OST0030: Connection restored to 21c8c585-5f1e-4 (at 10.50.8.18@o2ib2) [1276459.546660] Lustre: Skipped 5 previous similar messages [1277363.887312] Lustre: fir-OST0032: haven't heard from client 0b982012-b4a2-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f5aec00, cur 1585840787 expire 1585840637 last 1585840560 [1277363.907369] Lustre: Skipped 5 previous similar messages [1277452.459047] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1277452.467749] Lustre: Skipped 5 previous similar messages [1289939.862787] Lustre: fir-OST0034: haven't heard from client 74a22577-dbc5-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa005c0a95400, cur 1585853363 expire 1585853213 last 1585853136 [1289939.882955] Lustre: Skipped 5 previous similar messages [1289970.400451] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1289970.409240] Lustre: Skipped 5 previous similar messages [1290339.419289] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1290339.428079] Lustre: Skipped 5 previous similar messages [1290398.859910] Lustre: fir-OST0030: haven't heard from client 34f0ba71-4d3e-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febef4ce400, cur 1585853822 expire 1585853672 last 1585853595 [1290398.880055] Lustre: Skipped 5 previous similar messages [1290412.954742] LustreError: 49086:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.49.21.21@o2ib1) returned error from glimpse AST (req@ffff9fd5a21f6c00 x1661542610899328 status -107 rc -107), evict it ns: filter-fir-OST0038_UUID lock: ffff9fe5034a18c0/0xf3793efb5448b394 lrc: 3/0,0 mode: PW/PW res: [0x591a936:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) flags: 0x40000000020000 nid: 10.49.21.21@o2ib1 remote: 0x2248accdd48ebb6f expref: 5 pid: 50694 timeout: 0 lvb_type: 0 [1290412.999656] LustreError: 138-a: fir-OST0038: A client on nid 10.49.21.21@o2ib1 was evicted due to a lock glimpse callback time out: rc -107 [1290413.012400] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 1585853836s: evicting client at 10.49.21.21@o2ib1 ns: filter-fir-OST0038_UUID lock: ffff9fe5034a18c0/0xf3793efb5448b394 lrc: 3/0,0 mode: PW/PW res: [0x591a936:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) flags: 0x40000000020000 nid: 10.49.21.21@o2ib1 remote: 0x2248accdd48ebb6f expref: 6 pid: 50694 timeout: 0 lvb_type: 0 [1290892.858601] Lustre: fir-OST0030: haven't heard from client a6f39580-441f-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb6f11a400, cur 1585854316 expire 1585854166 last 1585854089 [1290892.878749] Lustre: Skipped 4 previous similar messages [1290960.211669] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1290960.220458] Lustre: Skipped 4 previous similar messages [1291794.308348] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1291794.317158] Lustre: Skipped 6 previous similar messages [1291839.859134] Lustre: fir-OST0032: haven't heard from client ab887596-2654-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa009f21bc400, cur 1585855263 expire 1585855113 last 1585855036 [1291839.879276] Lustre: Skipped 5 previous similar messages [1292728.312653] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1292728.321518] Lustre: Skipped 5 previous similar messages [1292748.856890] Lustre: fir-OST0038: haven't heard from client 7800f108-edbf-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb6f11f800, cur 1585856172 expire 1585856022 last 1585855945 [1292748.877041] Lustre: Skipped 5 previous similar messages [1292749.856328] Lustre: fir-OST003a: haven't heard from client 7800f108-edbf-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f574800, cur 1585856173 expire 1585856023 last 1585855946 [1292749.876465] Lustre: Skipped 3 previous similar messages [1293255.855446] Lustre: fir-OST0038: haven't heard from client 25db87c4-882a-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb78d15400, cur 1585856679 expire 1585856529 last 1585856452 [1293255.875589] Lustre: Skipped 1 previous similar message [1293507.546643] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1293507.555434] Lustre: Skipped 5 previous similar messages [1293828.249170] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1293828.257979] Lustre: Skipped 5 previous similar messages [1293885.854274] Lustre: fir-OST0036: haven't heard from client b4ff50cd-f32a-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5ec3c00, cur 1585857309 expire 1585857159 last 1585857082 [1293885.874443] Lustre: Skipped 5 previous similar messages [1295234.851769] Lustre: fir-OST0038: haven't heard from client ddb48759-c809-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff5ce4a2800, cur 1585858658 expire 1585858508 last 1585858431 [1295234.871929] Lustre: Skipped 5 previous similar messages [1295262.603337] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1295262.612129] Lustre: Skipped 5 previous similar messages [1295980.051656] Lustre: fir-OST0032: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1295980.060447] Lustre: Skipped 4 previous similar messages [1296016.852375] Lustre: fir-OST003a: haven't heard from client 4e2e9ff3-63e1-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fcbc9282400, cur 1585859440 expire 1585859290 last 1585859213 [1296016.872563] Lustre: Skipped 5 previous similar messages [1296921.280717] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1296921.289533] Lustre: Skipped 5 previous similar messages [1296934.849887] Lustre: fir-OST0034: haven't heard from client 40c9ab89-a8eb-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93132d800, cur 1585860358 expire 1585860208 last 1585860131 [1296934.870031] Lustre: Skipped 5 previous similar messages [1297775.848542] Lustre: fir-OST0030: haven't heard from client 0d59efc3-4437-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ebaaac00, cur 1585861199 expire 1585861049 last 1585860972 [1297775.868746] Lustre: Skipped 5 previous similar messages [1297776.474478] Lustre: fir-OST0038: haven't heard from client 0d59efc3-4437-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eb9d7400, cur 1585861199 expire 1585861049 last 1585860972 [1297776.494654] Lustre: Skipped 4 previous similar messages [1297803.605986] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1297803.614801] Lustre: Skipped 5 previous similar messages [1298733.850899] Lustre: fir-OST0036: haven't heard from client 74c0ee2c-8abf-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbe7089400, cur 1585862157 expire 1585862007 last 1585861930 [1298759.215679] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1298759.224471] Lustre: Skipped 4 previous similar messages [1299563.851813] Lustre: fir-OST0032: haven't heard from client 7f24d805-4b6c-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf6b61400, cur 1585862987 expire 1585862837 last 1585862760 [1299563.871982] Lustre: Skipped 5 previous similar messages [1299607.531292] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1299607.540086] Lustre: Skipped 6 previous similar messages [1300636.434666] Lustre: fir-OST0032: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1300636.434668] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1300636.452342] Lustre: Skipped 4 previous similar messages [1300637.851005] Lustre: fir-OST0032: haven't heard from client dae56af1-c8a1-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e52d5400, cur 1585864061 expire 1585863911 last 1585863834 [1300637.871230] Lustre: Skipped 5 previous similar messages [1301515.841892] Lustre: fir-OST003a: haven't heard from client 5582702c-e427-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff505567800, cur 1585864939 expire 1585864789 last 1585864712 [1301515.862055] Lustre: Skipped 5 previous similar messages [1301536.347623] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1301536.356412] Lustre: Skipped 5 previous similar messages [1302217.546196] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1302217.554995] Lustre: Skipped 5 previous similar messages [1302265.844519] Lustre: fir-OST0032: haven't heard from client a5eeac10-067e-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbe47ff400, cur 1585865689 expire 1585865539 last 1585865462 [1302265.864683] Lustre: Skipped 5 previous similar messages [1302940.271680] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1302940.271682] Lustre: fir-OST0032: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1302940.289356] Lustre: Skipped 4 previous similar messages [1302971.840712] Lustre: fir-OST0034: haven't heard from client e22e23c2-e0bf-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffa4d278c00, cur 1585866395 expire 1585866245 last 1585866168 [1302971.860871] Lustre: Skipped 5 previous similar messages [1303918.839130] Lustre: fir-OST003a: haven't heard from client 8dc15a3d-db81-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7722e800, cur 1585867342 expire 1585867192 last 1585867115 [1303918.859306] Lustre: Skipped 5 previous similar messages [1303981.583066] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1303981.591913] Lustre: Skipped 5 previous similar messages [1304834.838294] Lustre: fir-OST0034: haven't heard from client 5124239a-c213-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fcc048f5800, cur 1585868258 expire 1585868108 last 1585868031 [1304834.858439] Lustre: Skipped 5 previous similar messages [1304856.299351] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1304856.308136] Lustre: Skipped 5 previous similar messages [1305620.502202] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1305620.510986] Lustre: Skipped 5 previous similar messages [1305634.836680] Lustre: fir-OST0038: haven't heard from client c867a4b7-7672-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd3a29000, cur 1585869058 expire 1585868908 last 1585868831 [1305634.856821] Lustre: Skipped 5 previous similar messages [1306385.712797] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1306385.721589] Lustre: Skipped 5 previous similar messages [1306423.838257] Lustre: fir-OST0036: haven't heard from client 08103f19-de06-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934fd800, cur 1585869847 expire 1585869697 last 1585869620 [1306423.858400] Lustre: Skipped 5 previous similar messages [1307270.027348] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1307270.036144] Lustre: Skipped 4 previous similar messages [1307315.836706] Lustre: fir-OST0032: haven't heard from client 69fe9fc9-3326-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fddde028c00, cur 1585870739 expire 1585870589 last 1585870512 [1307315.856854] Lustre: Skipped 5 previous similar messages [1308136.001481] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1308136.010268] Lustre: Skipped 6 previous similar messages [1308174.849997] Lustre: fir-OST003a: haven't heard from client 926cf9a9-13e7-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb77158800, cur 1585871598 expire 1585871448 last 1585871371 [1308174.870164] Lustre: Skipped 5 previous similar messages [1308355.834454] Lustre: fir-OST0036: haven't heard from client 9fafd4c0-0b7b-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf94e2400, cur 1585871779 expire 1585871629 last 1585871552 [1308355.854551] Lustre: Skipped 5 previous similar messages [1308408.381720] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1308408.390469] Lustre: Skipped 5 previous similar messages [1308898.211868] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1308898.220659] Lustre: Skipped 5 previous similar messages [1308915.833466] Lustre: fir-OST0036: haven't heard from client c65fab57-35bd-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b6800, cur 1585872339 expire 1585872189 last 1585872112 [1308915.853639] Lustre: Skipped 5 previous similar messages [1309685.001475] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1309685.010273] Lustre: Skipped 5 previous similar messages [1309702.832760] Lustre: fir-OST003a: haven't heard from client 2aa88a3c-b748-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b0400, cur 1585873126 expire 1585872976 last 1585872899 [1309702.852940] Lustre: Skipped 5 previous similar messages [1310284.257822] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1310284.266608] Lustre: Skipped 5 previous similar messages [1310288.832241] Lustre: fir-OST0036: haven't heard from client 5fea1caa-a7bb-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5a8400, cur 1585873712 expire 1585873562 last 1585873485 [1310288.852403] Lustre: Skipped 5 previous similar messages [1310879.218809] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1310879.227599] Lustre: Skipped 5 previous similar messages [1310912.836136] Lustre: fir-OST0036: haven't heard from client 4fc7f937-8bf2-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b1800, cur 1585874336 expire 1585874186 last 1585874109 [1310912.856302] Lustre: Skipped 5 previous similar messages [1311783.834464] Lustre: fir-OST0032: haven't heard from client 3abc36c6-05e4-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febaa600c00, cur 1585875207 expire 1585875057 last 1585874980 [1311783.854607] Lustre: Skipped 5 previous similar messages [1311886.980909] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1311886.989695] Lustre: Skipped 5 previous similar messages [1312470.275122] Lustre: fir-OST0032: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1312470.283911] Lustre: Skipped 4 previous similar messages [1312490.829783] Lustre: fir-OST0038: haven't heard from client 457a238b-523a-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00767529000, cur 1585875914 expire 1585875764 last 1585875687 [1312490.849971] Lustre: Skipped 5 previous similar messages [1313098.829685] Lustre: fir-OST0038: haven't heard from client f8ebc65b-1d76-4 (at 10.49.21.13@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb924a0000, cur 1585876522 expire 1585876372 last 1585876295 [1313098.849853] Lustre: Skipped 5 previous similar messages [1313176.205276] Lustre: fir-OST0030: Connection restored to f8ebc65b-1d76-4 (at 10.49.21.13@o2ib1) [1313176.214063] Lustre: Skipped 6 previous similar messages [1313498.830438] Lustre: fir-OST0038: haven't heard from client cb3fb692-e931-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febaa600c00, cur 1585876922 expire 1585876772 last 1585876695 [1313498.850580] Lustre: Skipped 5 previous similar messages [1313581.758772] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1313581.767564] Lustre: Skipped 5 previous similar messages [1319059.363150] LustreError: 49364:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli b13df3fd-f426-4 claims 512000 GRANT, real grant 28672 [1319654.826250] Lustre: fir-OST0032: haven't heard from client 0317481a-dbc1-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff6edc49c00, cur 1585883078 expire 1585882928 last 1585882851 [1319654.846408] Lustre: Skipped 5 previous similar messages [1319721.795309] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1319721.804100] Lustre: Skipped 5 previous similar messages [1336903.817427] LustreError: 49206:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 15e5b565-193d-4 claims 28672 GRANT, real grant 0 [1338226.337830] Lustre: fir-OST0030: Connection restored to 06f6e79a-f25a-4 (at 10.49.23.21@o2ib1) [1338226.346617] Lustre: Skipped 5 previous similar messages [1338285.809974] Lustre: fir-OST003a: haven't heard from client 06f6e79a-f25a-4 (at 10.49.23.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7eae3800, cur 1585901709 expire 1585901559 last 1585901482 [1338285.830112] Lustre: Skipped 5 previous similar messages [1338286.328831] Lustre: fir-OST0036: haven't heard from client 06f6e79a-f25a-4 (at 10.49.23.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d29f800, cur 1585901709 expire 1585901559 last 1585901482 [1349055.211330] LustreError: 49246:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli e0104642-d336-4 claims 28672 GRANT, real grant 0 [1350784.345085] LustreError: 49241:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli e0104642-d336-4 claims 73728 GRANT, real grant 0 [1357175.936696] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1357175.945493] Lustre: Skipped 5 previous similar messages [1357204.775373] Lustre: fir-OST0032: haven't heard from client e11b8b3f-72d0-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b9e7dac00, cur 1585920628 expire 1585920478 last 1585920401 [1357204.795540] Lustre: Skipped 4 previous similar messages [1357910.357783] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1357910.366570] Lustre: Skipped 5 previous similar messages [1357954.779660] Lustre: fir-OST0036: haven't heard from client d474ebd5-a9fc-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbeae4b800, cur 1585921378 expire 1585921228 last 1585921151 [1357954.799812] Lustre: Skipped 5 previous similar messages [1358865.770138] Lustre: fir-OST0032: haven't heard from client e2f38f60-659f-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b1800, cur 1585922289 expire 1585922139 last 1585922062 [1358865.790299] Lustre: Skipped 5 previous similar messages [1358878.481751] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1358878.490539] Lustre: Skipped 5 previous similar messages [1359583.195264] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1359583.204076] Lustre: Skipped 5 previous similar messages [1359632.769097] Lustre: fir-OST0034: haven't heard from client 46b016f2-d1f4-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bce6d3400, cur 1585923056 expire 1585922906 last 1585922829 [1359632.789237] Lustre: Skipped 5 previous similar messages [1359642.814634] Lustre: fir-OST003a: haven't heard from client 46b016f2-d1f4-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffa4d27d400, cur 1585923066 expire 1585922916 last 1585922839 [1359642.834803] Lustre: Skipped 4 previous similar messages [1359708.768453] Lustre: fir-OST0034: haven't heard from client 681deff3-eb13-4 (at 10.50.6.54@o2ib2) in 204 seconds. I think it's dead, and I am evicting it. exp ffffa007ad30b000, cur 1585923132 expire 1585922982 last 1585922928 [1359731.780369] Lustre: fir-OST003a: haven't heard from client 681deff3-eb13-4 (at 10.50.6.54@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b1400, cur 1585923155 expire 1585923005 last 1585922928 [1359731.800465] Lustre: Skipped 4 previous similar messages [1359782.606829] Lustre: fir-OST0030: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1359782.615536] Lustre: Skipped 5 previous similar messages [1360512.768292] Lustre: fir-OST0034: haven't heard from client 7d2ba658-0baa-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa005c0a91c00, cur 1585923936 expire 1585923786 last 1585923709 [1360528.780863] Lustre: fir-OST003a: haven't heard from client 7d2ba658-0baa-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8ea5a9c00, cur 1585923952 expire 1585923802 last 1585923725 [1360528.801007] Lustre: Skipped 4 previous similar messages [1360576.465895] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1360576.474682] Lustre: Skipped 5 previous similar messages [1361355.772640] Lustre: fir-OST0030: haven't heard from client f0184f56-3555-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938ea9400, cur 1585924779 expire 1585924629 last 1585924552 [1361370.820863] Lustre: fir-OST0036: haven't heard from client f0184f56-3555-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff2e250c000, cur 1585924794 expire 1585924644 last 1585924567 [1361370.841011] Lustre: Skipped 4 previous similar messages [1361389.291381] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1361389.300166] Lustre: Skipped 5 previous similar messages [1362264.404018] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1362264.412810] Lustre: Skipped 5 previous similar messages [1362267.774923] Lustre: fir-OST0036: haven't heard from client 658cd963-8a05-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf9bb5000, cur 1585925691 expire 1585925541 last 1585925464 [1362272.366546] LustreError: 49060:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.49.21.21@o2ib1) returned error from glimpse AST (req@ffff9fd6aa527080 x1661543088718080 status -107 rc -107), evict it ns: filter-fir-OST0034_UUID lock: ffff9fee9b64dc40/0xf3793efb58aef448 lrc: 3/0,0 mode: PW/PW res: [0x58818ab:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) flags: 0x40000000020000 nid: 10.49.21.21@o2ib1 remote: 0xecd8bcdb829306e9 expref: 5 pid: 48998 timeout: 0 lvb_type: 0 [1362272.411417] LustreError: 138-a: fir-OST0034: A client on nid 10.49.21.21@o2ib1 was evicted due to a lock glimpse callback time out: rc -107 [1362272.424185] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 1585925695s: evicting client at 10.49.21.21@o2ib1 ns: filter-fir-OST0034_UUID lock: ffff9fee9b64dc40/0xf3793efb58aef448 lrc: 3/0,0 mode: PW/PW res: [0x58818ab:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) flags: 0x40000000020000 nid: 10.49.21.21@o2ib1 remote: 0xecd8bcdb829306e9 expref: 6 pid: 48998 timeout: 0 lvb_type: 0 [1363007.104876] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1363007.113659] Lustre: Skipped 5 previous similar messages [1363068.764961] Lustre: fir-OST0038: haven't heard from client e0d2a6f9-90c8-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bdf295400, cur 1585926492 expire 1585926342 last 1585926265 [1363068.785100] Lustre: Skipped 4 previous similar messages [1363859.135321] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1363859.135322] Lustre: fir-OST0034: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1363859.135325] Lustre: Skipped 1 previous similar message [1363859.158225] Lustre: Skipped 3 previous similar messages [1363886.766868] Lustre: fir-OST0038: haven't heard from client 67ebe6f5-5aae-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bef80f800, cur 1585927310 expire 1585927160 last 1585927083 [1363886.787038] Lustre: Skipped 5 previous similar messages [1364863.761737] Lustre: fir-OST003a: haven't heard from client 996bbcc9-9596-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd3a2a800, cur 1585928287 expire 1585928137 last 1585928060 [1364863.781919] Lustre: Skipped 5 previous similar messages [1364887.305355] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1364887.314151] Lustre: Skipped 5 previous similar messages [1365666.114290] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1365666.123075] Lustre: Skipped 5 previous similar messages [1365691.760083] Lustre: fir-OST0034: haven't heard from client 62b6f27c-0c27-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00b9e7dcc00, cur 1585929115 expire 1585928965 last 1585928888 [1365691.780242] Lustre: Skipped 5 previous similar messages [1366572.462116] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1366572.470902] Lustre: Skipped 5 previous similar messages [1366595.758010] Lustre: fir-OST003a: haven't heard from client 315777c7-4f4b-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5972800, cur 1585930019 expire 1585929869 last 1585929792 [1366595.778183] Lustre: Skipped 5 previous similar messages [1367396.376604] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1367396.385492] Lustre: Skipped 4 previous similar messages [1367408.758890] Lustre: fir-OST0038: haven't heard from client 1adc3b55-8749-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febaa601800, cur 1585930832 expire 1585930682 last 1585930605 [1367408.779080] Lustre: Skipped 5 previous similar messages [1367426.758707] Lustre: fir-OST0036: haven't heard from client 1adc3b55-8749-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa005c0a92800, cur 1585930850 expire 1585930700 last 1585930623 [1367484.755640] Lustre: fir-OST0038: haven't heard from client 68faa563-c819-4 (at 10.50.9.37@o2ib2) in 216 seconds. I think it's dead, and I am evicting it. exp ffffa00642e08c00, cur 1585930908 expire 1585930758 last 1585930692 [1367484.775711] Lustre: Skipped 9 previous similar messages [1367536.340810] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1367536.349516] Lustre: Skipped 5 previous similar messages [1368245.426763] Lustre: fir-OST0032: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1368245.426764] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1368245.444405] Lustre: Skipped 4 previous similar messages [1368275.811660] Lustre: fir-OST0030: haven't heard from client a9261f64-2c5e-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b4000, cur 1585931699 expire 1585931549 last 1585931472 [1369124.760746] Lustre: fir-OST0030: haven't heard from client a93e6deb-131a-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bfa66f800, cur 1585932548 expire 1585932398 last 1585932321 [1369124.780891] Lustre: Skipped 5 previous similar messages [1369150.502006] Lustre: fir-OST0032: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1369150.502007] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1369150.519589] Lustre: Skipped 4 previous similar messages [1369182.869618] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1369182.878322] Lustre: Skipped 5 previous similar messages [1369881.277838] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1369881.286629] Lustre: Skipped 5 previous similar messages [1369929.754660] Lustre: fir-OST003a: haven't heard from client f414c76d-7b01-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbed037800, cur 1585933353 expire 1585933203 last 1585933126 [1369929.774825] Lustre: Skipped 11 previous similar messages [1370839.750839] Lustre: fir-OST0036: haven't heard from client 0ec25327-8576-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f5a9c00, cur 1585934263 expire 1585934113 last 1585934036 [1370839.770915] Lustre: Skipped 5 previous similar messages [1370887.753212] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1370887.761912] Lustre: Skipped 5 previous similar messages [1370992.863306] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1370992.872185] Lustre: Skipped 5 previous similar messages [1371036.756730] Lustre: fir-OST0034: haven't heard from client a0f2210c-95b3-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfef14400, cur 1585934460 expire 1585934310 last 1585934233 [1371036.776905] Lustre: Skipped 5 previous similar messages [1372166.217379] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1372166.226171] Lustre: Skipped 5 previous similar messages [1372223.747946] Lustre: fir-OST003a: haven't heard from client 8eae559f-8716-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb91dba000, cur 1585935647 expire 1585935497 last 1585935420 [1372223.768091] Lustre: Skipped 5 previous similar messages [1373080.385238] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1373080.394021] Lustre: Skipped 4 previous similar messages [1373095.745957] Lustre: fir-OST0036: haven't heard from client 48846965-459c-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bcef7a800, cur 1585936519 expire 1585936369 last 1585936292 [1373095.766097] Lustre: Skipped 5 previous similar messages [1373528.768008] Lustre: fir-OST0032: haven't heard from client c1177a89-5194-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febb0522800, cur 1585936952 expire 1585936802 last 1585936725 [1373528.788072] Lustre: Skipped 5 previous similar messages [1373548.748470] Lustre: fir-OST0036: haven't heard from client c1177a89-5194-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febb0520000, cur 1585936972 expire 1585936822 last 1585936745 [1373586.675549] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1373586.684277] Lustre: Skipped 6 previous similar messages [1373708.876233] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1373708.885022] Lustre: Skipped 4 previous similar messages [1373759.743772] Lustre: fir-OST003a: haven't heard from client 304e6b7a-f74b-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbecd83800, cur 1585937183 expire 1585937033 last 1585936956 [1373759.763914] Lustre: Skipped 4 previous similar messages [1374665.096399] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1374665.105192] Lustre: Skipped 6 previous similar messages [1374713.742891] Lustre: fir-OST0038: haven't heard from client e26a0b8d-df3e-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf0ee8800, cur 1585938137 expire 1585937987 last 1585937910 [1374713.763034] Lustre: Skipped 5 previous similar messages [1375470.741082] Lustre: fir-OST003a: haven't heard from client 539d87e3-6b19-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febb0525000, cur 1585938894 expire 1585938744 last 1585938667 [1375470.761166] Lustre: Skipped 5 previous similar messages [1375513.685179] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1375513.693887] Lustre: Skipped 5 previous similar messages [1377546.734315] Lustre: fir-OST003a: haven't heard from client 835afd83-dfec-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ef1c000, cur 1585940970 expire 1585940820 last 1585940743 [1377546.754392] Lustre: Skipped 5 previous similar messages [1377592.662974] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1377592.671679] Lustre: Skipped 5 previous similar messages [1378184.009368] LustreError: 91660:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 15e5b565-193d-4 claims 16752640 GRANT, real grant 950272 [1380554.740611] Lustre: fir-OST003a: haven't heard from client 5ab99ad1-8b26-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7cd46400, cur 1585943978 expire 1585943828 last 1585943751 [1380554.760689] Lustre: Skipped 5 previous similar messages [1380781.623468] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1380781.632164] Lustre: Skipped 5 previous similar messages [1380813.737818] Lustre: fir-OST0030: haven't heard from client 7f196257-4021-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf6db8400, cur 1585944237 expire 1585944087 last 1585944010 [1380813.757987] Lustre: Skipped 5 previous similar messages [1380819.794420] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1380819.803252] Lustre: Skipped 5 previous similar messages [1380991.124432] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1380991.133224] Lustre: Skipped 5 previous similar messages [1381046.726839] Lustre: fir-OST0034: haven't heard from client 3f36eb03-79bd-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bd0bea000, cur 1585944470 expire 1585944320 last 1585944243 [1381046.746984] Lustre: Skipped 5 previous similar messages [1381166.108445] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1381166.117233] Lustre: Skipped 5 previous similar messages [1381243.740910] Lustre: fir-OST0032: haven't heard from client 274a7547-b7aa-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fee934fec00, cur 1585944667 expire 1585944517 last 1585944440 [1381243.761058] Lustre: Skipped 5 previous similar messages [1382879.884147] LustreError: 48339:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0032: cli 75520abd-92af-4 claims 16752640 GRANT, real grant 1441792 [1382903.741232] Lustre: fir-OST0038: haven't heard from client 73f503d3-884e-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00862740400, cur 1585946327 expire 1585946177 last 1585946100 [1382903.761315] Lustre: Skipped 5 previous similar messages [1382915.730819] Lustre: fir-OST0030: haven't heard from client 73f503d3-884e-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f570000, cur 1585946339 expire 1585946189 last 1585946112 [1382915.750885] Lustre: Skipped 4 previous similar messages [1382965.923478] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1382965.932183] Lustre: Skipped 5 previous similar messages [1384673.721803] Lustre: fir-OST0034: haven't heard from client 75373d48-fe8f-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd8f69f2400, cur 1585948097 expire 1585947947 last 1585947870 [1384715.399273] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1384715.407973] Lustre: Skipped 5 previous similar messages [1388304.716262] Lustre: fir-OST0036: haven't heard from client 69cc2ce3-9c83-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb78d14c00, cur 1585951728 expire 1585951578 last 1585951501 [1388304.736358] Lustre: Skipped 5 previous similar messages [1388363.420943] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1388363.429646] Lustre: Skipped 5 previous similar messages [1391254.753281] LustreError: 49360:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli b13df3fd-f426-4 claims 28672 GRANT, real grant 0 [1393374.679200] LustreError: 49312:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3e41d262-577a-4 claims 28672 GRANT, real grant 0 [1414757.680200] Lustre: fir-OST0038: haven't heard from client 037ca474-0477-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb6f11dc00, cur 1585978181 expire 1585978031 last 1585977954 [1414757.700489] Lustre: Skipped 5 previous similar messages [1414807.036117] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1414807.044816] Lustre: Skipped 5 previous similar messages [1427397.657853] Lustre: fir-OST003a: haven't heard from client cc32a056-ed4a-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb7b497400, cur 1585990821 expire 1585990671 last 1585990594 [1427397.677927] Lustre: Skipped 5 previous similar messages [1427428.532553] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1427428.541253] Lustre: Skipped 5 previous similar messages [1428884.675795] Lustre: fir-OST0036: haven't heard from client e4687612-d8e5-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fcc0265d400, cur 1585992308 expire 1585992158 last 1585992081 [1428884.695914] Lustre: Skipped 5 previous similar messages [1428923.616584] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1428923.625293] Lustre: Skipped 5 previous similar messages [1436301.532514] Lustre: fir-OST0030: Connection restored to bb05bfe6-c379-4 (at 10.50.4.28@o2ib2) [1436301.541221] Lustre: Skipped 5 previous similar messages [1436322.635390] Lustre: fir-OST0038: haven't heard from client bb05bfe6-c379-4 (at 10.50.4.28@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb92669c00, cur 1585999746 expire 1585999596 last 1585999519 [1436322.655448] Lustre: Skipped 5 previous similar messages [1436323.405372] Lustre: fir-OST0036: haven't heard from client bb05bfe6-c379-4 (at 10.50.4.28@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7d2f0c00, cur 1585999746 expire 1585999596 last 1585999519 [1436323.425428] Lustre: Skipped 4 previous similar messages [1440898.001647] Lustre: fir-OST0030: Connection restored to bb05bfe6-c379-4 (at 10.50.4.28@o2ib2) [1440898.010349] Lustre: Skipped 5 previous similar messages [1440941.626476] Lustre: fir-OST0038: haven't heard from client 9b7d199c-0123-4 (at 10.50.4.28@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feba46f4000, cur 1586004365 expire 1586004215 last 1586004138 [1442774.622668] Lustre: fir-OST0032: haven't heard from client 934ddb45-94a2-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff505567400, cur 1586006198 expire 1586006048 last 1586005971 [1442774.642853] Lustre: Skipped 5 previous similar messages [1442811.697568] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1442811.706265] Lustre: Skipped 3 previous similar messages [1443520.287124] Lustre: fir-OST0032: Connection restored to 1c46010b-e86f-4 (at 10.50.14.3@o2ib2) [1443520.295825] Lustre: Skipped 4 previous similar messages [1443580.621052] Lustre: fir-OST0032: haven't heard from client 68109525-701f-4 (at 10.50.14.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7ef98800, cur 1586007004 expire 1586006854 last 1586006777 [1443580.641156] Lustre: Skipped 5 previous similar messages [1447472.613047] Lustre: fir-OST0032: haven't heard from client 3dd153d6-2947-4 (at 10.50.4.28@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fddde02b000, cur 1586010896 expire 1586010746 last 1586010669 [1447472.633108] Lustre: Skipped 5 previous similar messages [1447862.713158] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1447862.721954] Lustre: Skipped 5 previous similar messages [1447900.612253] Lustre: fir-OST0038: haven't heard from client 0cb0db66-c8ed-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e59c1c00, cur 1586011324 expire 1586011174 last 1586011097 [1447900.632404] Lustre: Skipped 5 previous similar messages [1448441.610597] Lustre: fir-OST003a: haven't heard from client 9b0c99d2-345c-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5ec4800, cur 1586011865 expire 1586011715 last 1586011638 [1448441.630821] Lustre: Skipped 5 previous similar messages [1448490.877938] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1448490.886732] Lustre: Skipped 4 previous similar messages [1448828.385470] Lustre: fir-OST0030: Connection restored to 1c46010b-e86f-4 (at 10.50.14.3@o2ib2) [1448828.394287] Lustre: Skipped 5 previous similar messages [1448865.612510] Lustre: fir-OST003a: haven't heard from client ca6aa2c0-a51a-4 (at 10.50.14.3@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d0000, cur 1586012289 expire 1586012139 last 1586012062 [1448865.632586] Lustre: Skipped 5 previous similar messages [1448914.889982] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1448914.899155] Lustre: Skipped 5 previous similar messages [1448941.610421] Lustre: fir-OST0038: haven't heard from client 33c56caf-f05e-4 (at 10.49.21.21@o2ib1) in 199 seconds. I think it's dead, and I am evicting it. exp ffffa00befa7d400, cur 1586012365 expire 1586012215 last 1586012166 [1448941.630561] Lustre: Skipped 5 previous similar messages [1448969.614849] Lustre: fir-OST0030: haven't heard from client 33c56caf-f05e-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d5800, cur 1586012393 expire 1586012243 last 1586012166 [1448969.634994] Lustre: Skipped 3 previous similar messages [1451153.153783] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1451153.162573] Lustre: Skipped 4 previous similar messages [1451174.613384] Lustre: fir-OST0030: haven't heard from client f072dd11-76e6-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bdf290800, cur 1586014598 expire 1586014448 last 1586014371 [1451174.633548] Lustre: Skipped 1 previous similar message [1451250.607245] Lustre: fir-OST003a: haven't heard from client 5a471ce2-27f1-4 (at 10.50.6.54@o2ib2) in 176 seconds. I think it's dead, and I am evicting it. exp ffff9ff53fb07800, cur 1586014674 expire 1586014524 last 1586014498 [1451250.627416] Lustre: Skipped 5 previous similar messages [1451398.158560] Lustre: fir-OST0032: Connection restored to eceee209-ec05-4 (at 10.50.6.54@o2ib2) [1451398.167260] Lustre: Skipped 6 previous similar messages [1453772.599313] Lustre: fir-OST0030: haven't heard from client d6539a1b-0c76-4 (at 10.49.7.8@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb800ca400, cur 1586017196 expire 1586017046 last 1586016969 [1453772.619288] Lustre: Skipped 5 previous similar messages [1453788.605547] Lustre: fir-OST003a: haven't heard from client d6539a1b-0c76-4 (at 10.49.7.8@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e8f0000, cur 1586017212 expire 1586017062 last 1586016985 [1453855.862824] Lustre: fir-OST0030: Connection restored to d6539a1b-0c76-4 (at 10.49.7.8@o2ib1) [1453855.871432] Lustre: Skipped 5 previous similar messages [1454665.600697] Lustre: fir-OST0032: haven't heard from client 97017afb-9fdb-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf5fed400, cur 1586018089 expire 1586017939 last 1586017862 [1454665.620791] Lustre: Skipped 4 previous similar messages [1454693.733938] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1454693.742638] Lustre: Skipped 5 previous similar messages [1465567.447911] Lustre: fir-OST0030: Connection restored to ee56fbe2-040d-4 (at 10.49.25.17@o2ib1) [1465567.456702] Lustre: Skipped 5 previous similar messages [1469981.573511] Lustre: fir-OST0032: haven't heard from client ff3d8c7f-ee13-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00862744400, cur 1586033405 expire 1586033255 last 1586033178 [1469981.593584] Lustre: Skipped 5 previous similar messages [1470016.977269] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1470016.985971] Lustre: Skipped 5 previous similar messages [1480509.539492] Lustre: fir-OST0038: haven't heard from client dfbfebbc-fbbb-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd49a269800, cur 1586043933 expire 1586043783 last 1586043706 [1480509.559567] Lustre: Skipped 5 previous similar messages [1480510.435212] Lustre: fir-OST0030: haven't heard from client dfbfebbc-fbbb-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf91c0c00, cur 1586043933 expire 1586043783 last 1586043706 [1480510.461478] Lustre: Skipped 1 previous similar message [1480546.490066] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1480546.498778] Lustre: Skipped 4 previous similar messages [1480547.241528] Lustre: fir-OST0032: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1495065.503123] Lustre: fir-OST0032: haven't heard from client 03fdb3f2-9fe5-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feba46f6400, cur 1586058489 expire 1586058339 last 1586058262 [1495065.523187] Lustre: Skipped 3 previous similar messages [1495133.436492] Lustre: fir-OST0030: Connection restored to 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) [1495133.445197] Lustre: Skipped 5 previous similar messages [1495371.497218] Lustre: fir-OST003a: haven't heard from client af8b075a-d960-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00bf8b5b800, cur 1586058795 expire 1586058645 last 1586058568 [1495371.517288] Lustre: Skipped 5 previous similar messages [1495397.489552] Lustre: fir-OST0034: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1495397.498266] Lustre: Skipped 5 previous similar messages [1495581.496478] Lustre: fir-OST003a: haven't heard from client 05c68320-d699-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00862744400, cur 1586059005 expire 1586058855 last 1586058778 [1495581.516539] Lustre: Skipped 5 previous similar messages [1495619.163299] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1495619.163299] Lustre: fir-OST0034: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1495619.186494] Lustre: Skipped 4 previous similar messages [1497679.491492] Lustre: fir-OST0034: haven't heard from client ce295894-cf8f-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfe454800, cur 1586061103 expire 1586060953 last 1586060876 [1497679.511550] Lustre: Skipped 5 previous similar messages [1497790.095835] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1497790.104538] Lustre: Skipped 5 previous similar messages [1499946.488617] Lustre: fir-OST003a: haven't heard from client 87e3819d-30dd-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbe7088800, cur 1586063370 expire 1586063220 last 1586063143 [1499946.508718] Lustre: Skipped 5 previous similar messages [1499983.687500] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1499983.696197] Lustre: Skipped 5 previous similar messages [1502318.478326] Lustre: fir-OST0032: haven't heard from client d497665e-1a35-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd49a269800, cur 1586065742 expire 1586065592 last 1586065515 [1502318.498394] Lustre: Skipped 5 previous similar messages [1502367.212754] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1502367.221456] Lustre: Skipped 5 previous similar messages [1509147.464370] Lustre: fir-OST003a: haven't heard from client 8d22f512-aa17-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa00595e8e400, cur 1586072571 expire 1586072421 last 1586072344 [1509147.484435] Lustre: Skipped 5 previous similar messages [1509190.276629] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1509190.285327] Lustre: Skipped 4 previous similar messages [1509543.462702] Lustre: fir-OST0030: haven't heard from client e51c3087-0e32-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febb0526400, cur 1586072967 expire 1586072817 last 1586072740 [1509543.482765] Lustre: Skipped 5 previous similar messages [1509612.934304] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1509612.943014] Lustre: Skipped 6 previous similar messages [1512277.458106] Lustre: fir-OST0030: haven't heard from client 04237cd7-b89c-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe3846e0000, cur 1586075701 expire 1586075551 last 1586075474 [1512277.478166] Lustre: Skipped 5 previous similar messages [1512284.459277] Lustre: fir-OST0036: haven't heard from client 04237cd7-b89c-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e6a9400, cur 1586075708 expire 1586075558 last 1586075481 [1512284.479374] Lustre: Skipped 4 previous similar messages [1512308.527231] Lustre: fir-OST0032: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1512308.535934] Lustre: Skipped 5 previous similar messages [1514567.451073] Lustre: fir-OST0032: haven't heard from client a734e045-6ae0-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd938eac000, cur 1586077991 expire 1586077841 last 1586077764 [1514590.395530] Lustre: fir-OST0032: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1514590.404236] Lustre: Skipped 5 previous similar messages [1520414.439157] Lustre: fir-OST0038: haven't heard from client 3ac90f8e-5504-4 (at 10.49.7.8@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f5aec00, cur 1586083838 expire 1586083688 last 1586083611 [1520414.459230] Lustre: Skipped 5 previous similar messages [1524312.430375] Lustre: fir-OST0034: haven't heard from client 71bbaa66-52fd-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fdbf91c4400, cur 1586087736 expire 1586087586 last 1586087509 [1524312.450433] Lustre: Skipped 5 previous similar messages [1524314.438243] Lustre: fir-OST0032: haven't heard from client 71bbaa66-52fd-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf1294c00, cur 1586087738 expire 1586087588 last 1586087511 [1524320.439522] Lustre: fir-OST0038: haven't heard from client 71bbaa66-52fd-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8e9512c00, cur 1586087744 expire 1586087594 last 1586087517 [1524320.459639] Lustre: Skipped 2 previous similar messages [1524334.423609] Lustre: fir-OST0036: haven't heard from client 71bbaa66-52fd-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa005c0a96000, cur 1586087758 expire 1586087608 last 1586087531 [1524375.274535] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1524375.283244] Lustre: Skipped 5 previous similar messages [1524604.807481] LustreError: 91684:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli b50e5f2f-f7a2-4 claims 28672 GRANT, real grant 0 [1527974.418688] Lustre: fir-OST0030: haven't heard from client 5c81c5e2-1ff2-4 (at 10.50.8.20@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd9366ce000, cur 1586091398 expire 1586091248 last 1586091171 [1528340.667783] Lustre: fir-OST0030: Connection restored to 84be5f83-bbde-4 (at 10.50.8.20@o2ib2) [1528340.676517] Lustre: Skipped 5 previous similar messages [1534894.796825] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586098311/real 1586098311] req@ffff9fed3f704800 x1661544942347136/t0(0) o104->fir-OST0034@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1586098318 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1534894.824395] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 3 previous similar messages [1534915.833776] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586098332/real 1586098332] req@ffff9fed3f704800 x1661544942347136/t0(0) o104->fir-OST0034@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1586098339 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1534915.861244] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [1534950.870709] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586098367/real 1586098367] req@ffff9fed3f704800 x1661544942347136/t0(0) o104->fir-OST0034@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1586098374 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1534950.898153] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 4 previous similar messages [1535020.908548] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586098437/real 1586098437] req@ffff9fed3f704800 x1661544942347136/t0(0) o104->fir-OST0034@10.50.9.27@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1586098444 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1535020.936013] Lustre: 49403:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 9 previous similar messages [1535041.945514] LustreError: 49403:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.50.9.27@o2ib2) failed to reply to blocking AST (req@ffff9fed3f704800 x1661544942347136 status 0 rc -110), evict it ns: filter-fir-OST0034_UUID lock: ffff9fcbc795e0c0/0xf3793efb68364135 lrc: 4/0,0 mode: PR/PR res: [0x1000000402:0xfa493f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->134217727) flags: 0x60000400010020 nid: 10.50.9.27@o2ib2 remote: 0xe1eb46d068e46b3 expref: 7 pid: 48976 timeout: 1535187 lvb_type: 1 [1535041.991574] LustreError: 138-a: fir-OST0034: A client on nid 10.50.9.27@o2ib2 was evicted due to a lock blocking callback time out: rc -110 [1535042.004318] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 154s: evicting client at 10.50.9.27@o2ib2 ns: filter-fir-OST0034_UUID lock: ffff9fcbc795e0c0/0xf3793efb68364135 lrc: 3/0,0 mode: PR/PR res: [0x1000000402:0xfa493f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->134217727) flags: 0x60000400010020 nid: 10.50.9.27@o2ib2 remote: 0xe1eb46d068e46b3 expref: 8 pid: 48976 timeout: 0 lvb_type: 1 [1535089.418532] Lustre: fir-OST003a: haven't heard from client 29f566cd-115e-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93854f000, cur 1586098513 expire 1586098363 last 1586098286 [1535089.438642] Lustre: Skipped 5 previous similar messages [1535110.402886] Lustre: fir-OST0032: haven't heard from client 29f566cd-115e-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd939a64c00, cur 1586098534 expire 1586098384 last 1586098307 [1535110.422961] Lustre: Skipped 2 previous similar messages [1535110.428464] LustreError: 48882:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.50.9.27@o2ib2) failed to reply to blocking AST (req@ffff9fdbf9b8da00 x1661544942851328 status 0 rc -5), evict it ns: filter-fir-OST0036_UUID lock: ffff9fd324de9680/0xf3793efb6836456b lrc: 4/0,0 mode: PR/PR res: [0x1080000402:0xfb2f0b:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->134217727) flags: 0x60000400010020 nid: 10.50.9.27@o2ib2 remote: 0xe1eb46d068e6b50 expref: 10 pid: 49012 timeout: 1535206 lvb_type: 1 [1535110.428471] LustreError: 138-a: fir-OST0036: A client on nid 10.50.9.27@o2ib2 was evicted due to a lock blocking callback time out: rc -5 [1535110.428473] LustreError: 138-a: fir-OST0036: A client on nid 10.50.9.27@o2ib2 was evicted due to a lock blocking callback time out: rc -5 [1535110.428473] LustreError: Skipped 1 previous similar message [1535110.428474] LustreError: Skipped 1 previous similar message [1535110.510894] LustreError: 48882:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) Skipped 3 previous similar messages [1535134.477398] Lustre: fir-OST0032: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1535134.486123] Lustre: Skipped 5 previous similar messages [1553506.363537] Lustre: fir-OST0034: haven't heard from client 90b6f92d-d059-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffb66b74000, cur 1586116930 expire 1586116780 last 1586116703 [1553506.383606] Lustre: Skipped 1 previous similar message [1553509.373207] Lustre: fir-OST0038: haven't heard from client 90b6f92d-d059-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf77b4800, cur 1586116933 expire 1586116783 last 1586116706 [1553525.370634] Lustre: fir-OST0030: haven't heard from client 90b6f92d-d059-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf0eea000, cur 1586116949 expire 1586116799 last 1586116722 [1553546.367460] Lustre: fir-OST0032: haven't heard from client 90b6f92d-d059-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd930bc5000, cur 1586116970 expire 1586116820 last 1586116743 [1553546.387521] Lustre: Skipped 1 previous similar message [1553768.277541] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1553768.286276] Lustre: Skipped 5 previous similar messages [1558762.196359] LustreError: 49360:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3da00d75-6c3b-4 claims 28672 GRANT, real grant 12288 [1559851.333652] Lustre: 48949:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for sent delay: [sent 1586123267/real 0] req@ffff9fd31fc38900 x1661545458676800/t0(0) o106->fir-OST003a@10.49.26.32@o2ib1:15/16 lens 296/280 e 0 to 1 dl 1586123274 ref 2 fl Rpc:X/0/ffffffff rc 0/-1 [1559851.360413] Lustre: 48949:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 15 previous similar messages [1559852.168535] Lustre: fir-OST0032: Client 8833809f-4c27-4 (at 10.49.18.17@o2ib1) reconnecting [1559852.177082] Lustre: fir-OST0032: Connection restored to 8833809f-4c27-4 (at 10.49.18.17@o2ib1) [1559853.238141] Lustre: fir-OST0036: Client 8833809f-4c27-4 (at 10.49.18.17@o2ib1) reconnecting [1559853.246686] Lustre: fir-OST0036: Connection restored to 8833809f-4c27-4 (at 10.49.18.17@o2ib1) [1559855.162562] Lustre: fir-OST0034: Client 14f6955a-e3d1-4 (at 10.49.8.32@o2ib1) reconnecting [1559855.171047] Lustre: fir-OST0034: Connection restored to 14f6955a-e3d1-4 (at 10.49.8.32@o2ib1) [1559858.030992] Lustre: fir-OST0030: Client 045ff6bc-9580-4 (at 10.49.17.25@o2ib1) reconnecting [1559858.039524] Lustre: Skipped 4 previous similar messages [1559858.044975] Lustre: fir-OST0030: Connection restored to 045ff6bc-9580-4 (at 10.49.17.25@o2ib1) [1559858.053799] Lustre: Skipped 4 previous similar messages [1559862.752341] Lustre: fir-OST003a: Client a5d32a00-dfb8-4 (at 10.49.18.20@o2ib1) reconnecting [1559862.760876] Lustre: Skipped 14 previous similar messages [1559862.766383] Lustre: fir-OST003a: Connection restored to a5d32a00-dfb8-4 (at 10.49.18.20@o2ib1) [1559862.775171] Lustre: Skipped 14 previous similar messages [1559873.375922] Lustre: fir-OST0032: Client 0f2d3178-05c2-4 (at 10.49.0.64@o2ib1) reconnecting [1559873.375923] Lustre: fir-OST0030: Client 0f2d3178-05c2-4 (at 10.49.0.64@o2ib1) reconnecting [1559873.375926] Lustre: Skipped 1 previous similar message [1559873.375944] Lustre: fir-OST0030: Connection restored to 0f2d3178-05c2-4 (at 10.49.0.64@o2ib1) [1559873.375945] Lustre: Skipped 1 previous similar message [1559873.412123] Lustre: Skipped 4 previous similar messages [1559878.592583] LustreError: 137-5: fir-OST0033_UUID: not available for connect from 10.49.18.17@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1559878.610175] LustreError: Skipped 1 previous similar message [1559881.474398] LustreError: 137-5: fir-OST0039_UUID: not available for connect from 10.49.18.31@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1559883.067328] LustreError: 137-5: fir-OST0033_UUID: not available for connect from 10.49.17.25@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1559887.827493] LustreError: 137-5: fir-OST0037_UUID: not available for connect from 10.49.21.19@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1559887.845080] LustreError: Skipped 1 previous similar message [1559892.191603] LustreError: 137-5: fir-OST0037_UUID: not available for connect from 10.49.8.32@o2ib1 (no target). If you are running an HA pair check that the target is mounted on the other server. [1559892.209083] LustreError: Skipped 1 previous similar message [1559903.680533] Lustre: fir-OST0032: Connection restored to 8833809f-4c27-4 (at 10.49.18.17@o2ib1) [1559903.689320] Lustre: Skipped 6 previous similar messages [1559905.788783] Lustre: fir-OST0038: Client b86e3638-886b-4 (at 10.49.22.1@o2ib1) reconnecting [1559905.797225] Lustre: Skipped 3 previous similar messages [1561834.175855] LustreError: 48892:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3da00d75-6c3b-4 claims 28672 GRANT, real grant 20480 [1564973.946118] LustreError: 49296:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 2f954eed-4b6c-4 claims 28672 GRANT, real grant 12288 [1577931.307675] Lustre: fir-OST003a: haven't heard from client 89b8930d-7d49-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff8e5ab3800, cur 1586141355 expire 1586141205 last 1586141128 [1577931.327743] Lustre: Skipped 1 previous similar message [1577969.448226] Lustre: fir-OST0032: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1577969.456925] Lustre: Skipped 18 previous similar messages [1581693.300142] Lustre: fir-OST003a: haven't heard from client fe9d3211-9fa0-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffffa0090f6d0800, cur 1586145117 expire 1586144967 last 1586144890 [1581693.320198] Lustre: Skipped 5 previous similar messages [1581736.629361] Lustre: fir-OST0032: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1581736.638070] Lustre: Skipped 4 previous similar messages [1605625.473659] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1605625.482449] Lustre: Skipped 6 previous similar messages [1605653.264533] Lustre: fir-OST003a: haven't heard from client 5b0cb4e2-0c62-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b6c00, cur 1586169077 expire 1586168927 last 1586168850 [1605653.284755] Lustre: Skipped 5 previous similar messages [1607170.992114] LustreError: 91653:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 4cb83747-634b-4 claims 28672 GRANT, real grant 24576 [1615060.244813] Lustre: fir-OST0038: haven't heard from client b9dc3cb9-ba1a-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fe8eaa84800, cur 1586178484 expire 1586178334 last 1586178257 [1615060.265002] Lustre: Skipped 5 previous similar messages [1615148.440558] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1615148.449352] Lustre: Skipped 5 previous similar messages [1616544.313408] Lustre: fir-OST0030: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1616544.322216] Lustre: Skipped 5 previous similar messages [1616580.243012] Lustre: fir-OST0034: haven't heard from client 5d276014-2b20-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fed6a5e5c00, cur 1586180004 expire 1586179854 last 1586179777 [1616580.263197] Lustre: Skipped 5 previous similar messages [1625897.713866] LustreError: 49111:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0036: cli 81a86e52-614e-4 claims 16752640 GRANT, real grant 512000 [1630558.217896] Lustre: fir-OST0034: haven't heard from client 3367b05d-b386-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd93854cc00, cur 1586193982 expire 1586193832 last 1586193755 [1630558.237987] Lustre: Skipped 5 previous similar messages [1630616.491223] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1630616.499926] Lustre: Skipped 3 previous similar messages [1630617.021844] Lustre: fir-OST0038: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1630617.030663] Lustre: Skipped 1 previous similar message [1638415.769354] Lustre: fir-OST0032: Connection restored to d6539a1b-0c76-4 (at 10.49.7.8@o2ib1) [1638415.777967] Lustre: Skipped 4 previous similar messages [1644572.712881] Lustre: fir-OST0030: Connection restored to bb05bfe6-c379-4 (at 10.50.4.28@o2ib2) [1644572.721581] Lustre: Skipped 4 previous similar messages [1645783.480481] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586209200/real 1586209200] req@ffff9fdbf9eeba80 x1661546535713600/t0(0) o106->fir-OST0034@10.49.0.63@o2ib1:15/16 lens 296/280 e 0 to 1 dl 1586209207 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 [1645783.507904] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 5 previous similar messages [1645790.517469] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586209207/real 1586209207] req@ffff9fdbf9eeba80 x1661546535713600/t0(0) o106->fir-OST0034@10.49.0.63@o2ib1:15/16 lens 296/280 e 0 to 1 dl 1586209214 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1645797.544443] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586209214/real 1586209214] req@ffff9fdbf9eeba80 x1661546535713600/t0(0) o106->fir-OST0034@10.49.0.63@o2ib1:15/16 lens 296/280 e 0 to 1 dl 1586209221 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1645811.571409] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586209228/real 1586209228] req@ffff9fdbf9eeba80 x1661546535713600/t0(0) o106->fir-OST0034@10.49.0.63@o2ib1:15/16 lens 296/280 e 0 to 1 dl 1586209235 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1645811.598861] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 1 previous similar message [1645832.608365] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1586209249/real 1586209249] req@ffff9fdbf9eeba80 x1661546535713600/t0(0) o106->fir-OST0034@10.49.0.63@o2ib1:15/16 lens 296/280 e 0 to 1 dl 1586209256 ref 1 fl Rpc:X/2/ffffffff rc 0/-1 [1645832.635817] Lustre: 49142:0:(client.c:2133:ptlrpc_expire_one_request()) Skipped 2 previous similar messages [1645845.744718] Lustre: fir-OST0030: Connection restored to 9823016a-d5f7-4 (at 10.49.0.63@o2ib1) [1645845.753420] Lustre: Skipped 5 previous similar messages [1645846.645507] LustreError: 49142:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.49.0.63@o2ib1) returned error from glimpse AST (req@ffff9fdbf9eeba80 x1661546535713600 status -107 rc -107), evict it ns: filter-fir-OST0034_UUID lock: ffff9ff4ed4a5340/0xf3793efb7b19c0f9 lrc: 3/0,0 mode: PW/PW res: [0x584c7cb:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 34359738368->18446744073709551615) flags: 0x40000000000000 nid: 10.49.0.63@o2ib1 remote: 0x39000aa898e5cfc2 expref: 21 pid: 49417 timeout: 0 lvb_type: 0 [1645846.692574] LustreError: 138-a: fir-OST0034: A client on nid 10.49.0.63@o2ib1 was evicted due to a lock glimpse callback time out: rc -107 [1645846.705180] LustreError: Skipped 1 previous similar message [1645846.710963] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 1586209270s: evicting client at 10.49.0.63@o2ib1 ns: filter-fir-OST0034_UUID lock: ffff9ff4ed4a5340/0xf3793efb7b19c0f9 lrc: 3/0,0 mode: PW/PW res: [0x584c7cb:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 34359738368->18446744073709551615) flags: 0x40000000000000 nid: 10.49.0.63@o2ib1 remote: 0x39000aa898e5cfc2 expref: 22 pid: 49417 timeout: 0 lvb_type: 0 [1645847.516152] LustreError: 49086:0:(ldlm_lockd.c:681:ldlm_handle_ast_error()) ### client (nid 10.49.0.63@o2ib1) returned error from glimpse AST (req@ffff9fcc017f6300 x1661546536272640 status -107 rc -107), evict it ns: filter-fir-OST003a_UUID lock: ffff9fe5f0190900/0xf3793efb7b19c7a5 lrc: 3/0,0 mode: PW/PW res: [0x1180000400:0x63525f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 67108864->68719476735) flags: 0x40000000000000 nid: 10.49.0.63@o2ib1 remote: 0x39000aa898e5d071 expref: 15 pid: 49417 timeout: 0 lvb_type: 0 [1645847.562820] LustreError: 138-a: fir-OST003a: A client on nid 10.49.0.63@o2ib1 was evicted due to a lock glimpse callback time out: rc -107 [1645847.575442] LustreError: 48281:0:(ldlm_lockd.c:256:expired_lock_main()) ### lock callback timer expired after 1586209271s: evicting client at 10.49.0.63@o2ib1 ns: filter-fir-OST003a_UUID lock: ffff9fe5f0190900/0xf3793efb7b19c7a5 lrc: 3/0,0 mode: PW/PW res: [0x1180000400:0x63525f:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 67108864->68719476735) flags: 0x40000000000000 nid: 10.49.0.63@o2ib1 remote: 0x39000aa898e5d071 expref: 16 pid: 49417 timeout: 0 lvb_type: 0 [1645897.186841] Lustre: fir-OST0038: haven't heard from client 9823016a-d5f7-4 (at 10.49.0.63@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9febf9c24c00, cur 1586209321 expire 1586209171 last 1586209094 [1645897.206902] Lustre: Skipped 5 previous similar messages [1705662.585668] Lustre: fir-OST0030: Connection restored to b3a54002-f8c5-4 (at 10.50.12.2@o2ib2) [1705662.594380] Lustre: Skipped 4 previous similar messages [1705684.088622] Lustre: fir-OST0034: haven't heard from client b3a54002-f8c5-4 (at 10.50.12.2@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7fd32800, cur 1586269108 expire 1586268958 last 1586268881 [1705684.108700] Lustre: Skipped 2 previous similar messages [1713268.607674] Lustre: fir-OST0032: Connection restored to b449688d-0d11-4 (at 10.49.21.21@o2ib1) [1713268.616496] Lustre: Skipped 5 previous similar messages [1713285.072925] Lustre: fir-OST003a: haven't heard from client 19ab4ca3-4eb4-4 (at 10.49.21.21@o2ib1) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbfe452000, cur 1586276709 expire 1586276559 last 1586276482 [1713285.093070] Lustre: Skipped 5 previous similar messages [1716795.491326] Lustre: fir-OST0030: Connection restored to b3a54002-f8c5-4 (at 10.50.12.2@o2ib2) [1716795.500030] Lustre: Skipped 5 previous similar messages [1718915.061958] Lustre: fir-OST0032: haven't heard from client 7c15878b-7f74-4 (at 10.50.9.27@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9fd937e1b000, cur 1586282339 expire 1586282189 last 1586282112 [1718915.082023] Lustre: Skipped 5 previous similar messages [1718949.252683] Lustre: fir-OST0030: Connection restored to 59748e26-ab50-4 (at 10.50.9.27@o2ib2) [1718949.261398] Lustre: Skipped 5 previous similar messages [1722129.895805] LustreError: 48331:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST003a: cli 75520abd-92af-4 claims 811008 GRANT, real grant 0 [1722669.612183] LustreError: 49384:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0038: cli ef304413-5f16-4 claims 28672 GRANT, real grant 0 [1723842.193868] LustreError: 49385:0:(tgt_grant.c:758:tgt_grant_check()) fir-OST0034: cli 3e41d262-577a-4 claims 1441792 GRANT, real grant 28672 [1737188.036104] Lustre: fir-OST003a: haven't heard from client eeb6444d-4e51-4 (at 10.50.16.6@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ffbf29bd800, cur 1586300612 expire 1586300462 last 1586300385 [1737188.056200] Lustre: Skipped 5 previous similar messages [1737210.329536] Lustre: fir-OST0032: Connection restored to 6c45c03c-4b15-4 (at 10.50.16.6@o2ib2) [1737210.338237] Lustre: Skipped 5 previous similar messages [1749068.018403] Lustre: fir-OST0032: haven't heard from client 2ad6fa60-c8a2-4 (at 10.50.9.37@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9ff95e8b6800, cur 1586312492 expire 1586312342 last 1586312265 [1749068.038468] Lustre: Skipped 5 previous similar messages [1749118.074334] Lustre: fir-OST0030: Connection restored to f4363950-d6c3-4 (at 10.50.9.37@o2ib2) [1749118.083036] Lustre: Skipped 5 previous similar messages [1751518.726158] INFO: task kswapd0:276 blocked for more than 120 seconds. [1751518.732778] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751518.740788] kswapd0 D ffff9fdbf9d74100 0 276 2 0x00000000 [1751518.748096] Call Trace: [1751518.750732] [] ? enqueue_entity+0x2ef/0xbe0 [1751518.756778] [] schedule+0x29/0x70 [1751518.761950] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751518.769035] [] ? wake_up_atomic_t+0x30/0x30 [1751518.775088] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751518.782391] [] start_this_handle+0x1a1/0x430 [jbd2] [1751518.789096] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751518.795317] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751518.802119] [] ? ldiskfs_release_dquot+0x5c/0xa0 [ldiskfs] [1751518.809470] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751518.817072] [] ldiskfs_release_dquot+0x5c/0xa0 [ldiskfs] [1751518.824232] [] dqput+0xbd/0x1f0 [1751518.829216] [] __dquot_drop+0x85/0xc0 [1751518.834723] [] dquot_drop+0x35/0x40 [1751518.840067] [] ldiskfs_clear_inode+0x22/0x90 [ldiskfs] [1751518.847034] [] ldiskfs_evict_inode+0x9f/0x630 [ldiskfs] [1751518.854100] [] evict+0xb4/0x180 [1751518.859095] [] dispose_list+0x3e/0x50 [1751518.864588] [] prune_icache_sb+0x17c/0x330 [1751518.870517] [] prune_super+0x173/0x190 [1751518.876097] [] shrink_slab+0x175/0x340 [1751518.881695] [] ? vmpressure+0x21/0x90 [1751518.887205] [] balance_pgdat+0x4b1/0x5e0 [1751518.892970] [] kswapd+0x173/0x440 [1751518.898136] [] ? wake_up_atomic_t+0x30/0x30 [1751518.904172] [] ? balance_pgdat+0x5e0/0x5e0 [1751518.910113] [] kthread+0xd1/0xe0 [1751518.915199] [] ? insert_kthread_work+0x40/0x40 [1751518.921484] [] ret_from_fork_nospec_begin+0xe/0x21 [1751518.928125] [] ? insert_kthread_work+0x40/0x40 [1751518.934559] INFO: task jbd2/md4-8:48176 blocked for more than 120 seconds. [1751518.941625] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751518.949647] jbd2/md4-8 D ffff9febeba1d140 0 48176 2 0x00000080 [1751518.956937] Call Trace: [1751518.959567] [] schedule+0x29/0x70 [1751518.964737] [] jbd2_journal_commit_transaction+0x23c/0x19b0 [jbd2] [1751518.972754] [] ? dequeue_task_fair+0x41e/0x660 [1751518.979042] [] ? __switch_to+0xce/0x580 [1751518.984722] [] ? wake_up_atomic_t+0x30/0x30 [1751518.990753] [] ? __schedule+0x42a/0x860 [1751518.996434] [] ? try_to_del_timer_sync+0x5e/0x90 [1751519.002909] [] kjournald2+0xc9/0x260 [jbd2] [1751519.008936] [] ? wake_up_atomic_t+0x30/0x30 [1751519.014976] [] ? commit_timeout+0x10/0x10 [jbd2] [1751519.021439] [] kthread+0xd1/0xe0 [1751519.026528] [] ? insert_kthread_work+0x40/0x40 [1751519.032800] [] ret_from_fork_nospec_begin+0xe/0x21 [1751519.039446] [] ? insert_kthread_work+0x40/0x40 [1751519.045738] INFO: task ll_ost00_003:48312 blocked for more than 120 seconds. [1751519.052994] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751519.061015] ll_ost00_003 D ffff9fdbe0a81040 0 48312 2 0x00000080 [1751519.068310] Call Trace: [1751519.070941] [] schedule+0x29/0x70 [1751519.076122] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751519.083196] [] ? wake_up_atomic_t+0x30/0x30 [1751519.089238] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751519.096490] [] ? kiblnd_send+0x357/0xa20 [ko2iblnd] [1751519.103226] [] start_this_handle+0x1a1/0x430 [jbd2] [1751519.109981] [] ? lu_obj_hop_keycmp+0x1d/0x30 [obdclass] [1751519.117068] [] ? cfs_hash_bd_lookup_intent+0x63/0x170 [libcfs] [1751519.124747] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751519.130956] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751519.137787] [] ? ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1751519.145134] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751519.152712] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1751519.159871] [] dqget+0x3fa/0x450 [1751519.164947] [] dquot_get_dqblk+0x14/0x1f0 [1751519.170801] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1751519.178480] [] lquotactl_slv+0x27d/0xa00 [lquota] [1751519.185039] [] ofd_quotactl+0x13c/0x380 [ofd] [1751519.191285] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751519.198409] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751519.206182] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751519.213473] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751519.221355] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751519.228341] [] ? __wake_up+0x44/0x50 [1751519.233789] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751519.240298] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751519.247891] [] kthread+0xd1/0xe0 [1751519.252969] [] ? insert_kthread_work+0x40/0x40 [1751519.259256] [] ret_from_fork_nospec_begin+0xe/0x21 [1751519.265888] [] ? insert_kthread_work+0x40/0x40 [1751519.272165] INFO: task ll_ost_io00_000:48326 blocked for more than 120 seconds. [1751519.279669] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751519.287689] ll_ost_io00_000 D ffff9febcf2fe180 0 48326 2 0x00000080 [1751519.294988] Call Trace: [1751519.297632] [] schedule+0x29/0x70 [1751519.302805] [] osd_trans_stop+0x265/0x8e0 [osd_ldiskfs] [1751519.309881] [] ? ldiskfs_dirty_inode+0x54/0x60 [ldiskfs] [1751519.317023] [] ? wake_up_atomic_t+0x30/0x30 [1751519.323061] [] ofd_trans_stop+0x25/0x60 [ofd] [1751519.329289] [] ofd_commitrw_write+0x9d4/0x1db0 [ofd] [1751519.336093] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751519.342328] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751519.349069] [] ? lustre_msg_buf+0x17/0x60 [ptlrpc] [1751519.355744] [] ? __req_capsule_get+0x163/0x740 [ptlrpc] [1751519.362821] [] ? __enqueue_entity+0x78/0x80 [1751519.368860] [] ? target_send_reply_msg+0x170/0x170 [ptlrpc] [1751519.376313] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751519.383425] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751519.391184] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751519.398446] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751519.406338] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751519.413319] [] ? __wake_up+0x44/0x50 [1751519.418780] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751519.425270] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751519.432845] [] kthread+0xd1/0xe0 [1751519.437923] [] ? insert_kthread_work+0x40/0x40 [1751519.444222] [] ret_from_fork_nospec_begin+0xe/0x21 [1751519.450846] [] ? insert_kthread_work+0x40/0x40 [1751519.457152] INFO: task ll_ost_io00_001:48327 blocked for more than 120 seconds. [1751519.464642] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751519.472682] ll_ost_io00_001 D ffff9febcf2fc100 0 48327 2 0x00000080 [1751519.479991] Call Trace: [1751519.482634] [] schedule+0x29/0x70 [1751519.487808] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751519.494885] [] ? wake_up_atomic_t+0x30/0x30 [1751519.500904] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751519.508175] [] start_this_handle+0x1a1/0x430 [jbd2] [1751519.514894] [] ? osd_declare_xattr_set+0xf1/0x3a0 [osd_ldiskfs] [1751519.522660] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751519.528847] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751519.535645] [] ? osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751519.543019] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751519.550595] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751519.557766] [] ofd_trans_start+0x75/0xf0 [ofd] [1751519.564057] [] ofd_object_punch+0x798/0xd90 [ofd] [1751519.570612] [] ofd_punch_hdl+0x4f3/0xa80 [ofd] [1751519.576952] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751519.584057] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751519.591820] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751519.599089] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751519.606966] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751519.613952] [] ? __wake_up+0x44/0x50 [1751519.619403] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751519.625900] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751519.633492] [] kthread+0xd1/0xe0 [1751519.638583] [] ? insert_kthread_work+0x40/0x40 [1751519.644875] [] ret_from_fork_nospec_begin+0xe/0x21 [1751519.651518] [] ? insert_kthread_work+0x40/0x40 [1751519.657805] INFO: task ll_ost_io01_001:48331 blocked for more than 120 seconds. [1751519.665312] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751519.673340] ll_ost_io01_001 D ffff9febcf2fa080 0 48331 2 0x00000080 [1751519.680627] Call Trace: [1751519.683259] [] schedule+0x29/0x70 [1751519.688435] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751519.695507] [] ? wake_up_atomic_t+0x30/0x30 [1751519.701557] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751519.708806] [] start_this_handle+0x1a1/0x430 [jbd2] [1751519.715554] [] ? osd_declare_xattr_set+0xf1/0x3a0 [osd_ldiskfs] [1751519.723316] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751519.729514] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751519.736346] [] ? osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751519.743666] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751519.751267] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751519.758433] [] ofd_trans_start+0x75/0xf0 [ofd] [1751519.764710] [] ofd_object_punch+0x798/0xd90 [ofd] [1751519.771277] [] ofd_punch_hdl+0x4f3/0xa80 [ofd] [1751519.777607] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751519.784707] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751519.792476] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751519.799750] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751519.807610] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751519.814598] [] ? __wake_up+0x44/0x50 [1751519.820044] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751519.826538] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751519.834131] [] kthread+0xd1/0xe0 [1751519.839212] [] ? insert_kthread_work+0x40/0x40 [1751519.845500] [] ret_from_fork_nospec_begin+0xe/0x21 [1751519.852139] [] ? insert_kthread_work+0x40/0x40 [1751519.858427] INFO: task ll_ost_io03_001:48340 blocked for more than 120 seconds. [1751519.865930] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751519.873946] ll_ost_io03_001 D ffff9febf196d140 0 48340 2 0x00000080 [1751519.881235] Call Trace: [1751519.883889] [] ? __read_extent_tree_block+0x55/0x1e0 [ldiskfs] [1751519.891565] [] schedule+0x29/0x70 [1751519.896725] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751519.903801] [] ? wake_up_atomic_t+0x30/0x30 [1751519.909829] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751519.917085] [] start_this_handle+0x1a1/0x430 [jbd2] [1751519.923889] [] ? osd_declare_xattr_set+0xf1/0x3a0 [osd_ldiskfs] [1751519.931637] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751519.937841] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751519.944642] [] ? osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751519.951997] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751519.959596] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751519.966762] [] ofd_trans_start+0x75/0xf0 [ofd] [1751519.973040] [] ofd_commitrw_write+0xa31/0x1db0 [ofd] [1751519.979854] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751519.986095] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751519.992840] [] ? lustre_msg_buf+0x17/0x60 [ptlrpc] [1751519.999495] [] ? __req_capsule_get+0x163/0x740 [ptlrpc] [1751520.006584] [] ? class_handle2object+0xb9/0x1c0 [obdclass] [1751520.013963] [] ? update_curr+0x14c/0x1e0 [1751520.019717] [] ? mutex_lock+0x12/0x2f [1751520.025261] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751520.032362] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751520.040125] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751520.047431] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751520.055299] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751520.062285] [] ? __wake_up+0x44/0x50 [1751520.067743] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751520.074223] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751520.081813] [] kthread+0xd1/0xe0 [1751520.086871] [] ? insert_kthread_work+0x40/0x40 [1751520.093153] [] ret_from_fork_nospec_begin+0xe/0x21 [1751520.099804] [] ? insert_kthread_work+0x40/0x40 [1751520.106090] INFO: task ll_ost00_006:48815 blocked for more than 120 seconds. [1751520.113357] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751520.121363] ll_ost00_006 D ffff9ffbcc31e180 0 48815 2 0x00000080 [1751520.128659] Call Trace: [1751520.131304] [] schedule+0x29/0x70 [1751520.136455] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751520.143510] [] ? wake_up_atomic_t+0x30/0x30 [1751520.149527] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751520.156798] [] ? kiblnd_send+0x357/0xa20 [ko2iblnd] [1751520.163505] [] start_this_handle+0x1a1/0x430 [jbd2] [1751520.170256] [] ? lu_obj_hop_keycmp+0x1d/0x30 [obdclass] [1751520.177335] [] ? cfs_hash_bd_lookup_intent+0x63/0x170 [libcfs] [1751520.185018] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751520.191218] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751520.198040] [] ? ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1751520.205376] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751520.212981] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1751520.220125] [] dqget+0x3fa/0x450 [1751520.225201] [] dquot_get_dqblk+0x14/0x1f0 [1751520.231053] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1751520.238741] [] lquotactl_slv+0x27d/0xa00 [lquota] [1751520.245297] [] ofd_quotactl+0x13c/0x380 [ofd] [1751520.251544] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751520.258640] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751520.266409] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751520.273676] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751520.281562] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751520.288545] [] ? __wake_up+0x44/0x50 [1751520.294007] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751520.300499] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751520.308098] [] kthread+0xd1/0xe0 [1751520.313173] [] ? insert_kthread_work+0x40/0x40 [1751520.319464] [] ret_from_fork_nospec_begin+0xe/0x21 [1751520.326081] [] ? insert_kthread_work+0x40/0x40 [1751520.332375] INFO: task ll_ost00_008:48826 blocked for more than 120 seconds. [1751520.339613] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751520.347635] ll_ost00_008 D ffff9fcd296a8000 0 48826 2 0x00000080 [1751520.354923] Call Trace: [1751520.357558] [] schedule+0x29/0x70 [1751520.362715] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751520.369772] [] ? wake_up_atomic_t+0x30/0x30 [1751520.375792] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751520.383073] [] start_this_handle+0x1a1/0x430 [jbd2] [1751520.389790] [] ? osd_declare_write+0x350/0x490 [osd_ldiskfs] [1751520.397279] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751520.403500] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751520.410315] [] ? osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751520.417666] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751520.425260] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751520.432436] [] ofd_trans_start+0x75/0xf0 [ofd] [1751520.438724] [] ofd_attr_set+0x4b3/0xb90 [ofd] [1751520.444932] [] ofd_setattr_hdl+0x31d/0x940 [ofd] [1751520.451413] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751520.458515] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751520.466286] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751520.473557] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751520.481422] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751520.488412] [] ? __wake_up+0x44/0x50 [1751520.493860] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751520.500359] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751520.507947] [] kthread+0xd1/0xe0 [1751520.513024] [] ? insert_kthread_work+0x40/0x40 [1751520.519311] [] ret_from_fork_nospec_begin+0xe/0x21 [1751520.525946] [] ? insert_kthread_work+0x40/0x40 [1751520.532235] INFO: task ll_ost01_009:48829 blocked for more than 120 seconds. [1751520.539483] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message. [1751520.547505] ll_ost01_009 D ffff9ffbcc319040 0 48829 2 0x00000080 [1751520.554817] Call Trace: [1751520.557466] [] schedule+0x29/0x70 [1751520.562629] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751520.569702] [] ? wake_up_atomic_t+0x30/0x30 [1751520.575741] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751520.582977] [] start_this_handle+0x1a1/0x430 [jbd2] [1751520.589717] [] ? osd_declare_write+0x350/0x490 [osd_ldiskfs] [1751520.597221] [] ? kmem_cache_alloc+0x1c2/0x1f0 [1751520.603414] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751520.610250] [] ? osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751520.617568] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751520.625188] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751520.632332] [] ofd_trans_start+0x75/0xf0 [ofd] [1751520.638641] [] ofd_destroy+0x5d0/0x960 [ofd] [1751520.644755] [] ofd_destroy_by_fid+0x1e8/0x510 [ofd] [1751520.651520] [] ? ldlm_blocking_ast_nocheck+0x310/0x310 [ptlrpc] [1751520.659308] [] ? ldlm_expired_completion_wait+0x220/0x220 [ptlrpc] [1751520.667337] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1751520.673820] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751520.680928] [] ? ptlrpc_nrs_req_get_nolock0+0xd1/0x170 [ptlrpc] [1751520.688700] [] ? ktime_get_real_seconds+0xe/0x10 [libcfs] [1751520.695975] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751520.703852] [] ? ptlrpc_wait_event+0xa5/0x360 [ptlrpc] [1751520.710839] [] ? queued_spin_lock_slowpath+0xb/0xf [1751520.717476] [] ? _raw_spin_lock+0x20/0x30 [1751520.723390] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751520.729884] [] ? ptlrpc_register_service+0xf80/0xf80 [ptlrpc] [1751520.737488] [] kthread+0xd1/0xe0 [1751520.742562] [] ? insert_kthread_work+0x40/0x40 [1751520.748852] [] ret_from_fork_nospec_begin+0xe/0x21 [1751520.755485] [] ? insert_kthread_work+0x40/0x40 [1751567.745098] LNet: Service thread pid 56061 was inactive for 200.04s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1751567.762208] Pid: 56061, comm: ll_ost_io01_072 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751567.773113] Call Trace: [1751567.775773] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751567.782857] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751567.790142] [] start_this_handle+0x1a1/0x430 [jbd2] [1751567.796880] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751567.803749] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751567.811361] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751567.818568] [] ofd_trans_start+0x75/0xf0 [ofd] [1751567.824880] [] ofd_commitrw_write+0xa31/0x1db0 [ofd] [1751567.831702] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751567.837920] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751567.844711] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751567.851876] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751567.859774] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751567.866313] [] kthread+0xd1/0xe0 [1751567.871434] [] ret_from_fork_nospec_begin+0xe/0x21 [1751567.878102] [] 0xffffffffffffffff [1751567.883319] LustreError: dumping log to /tmp/lustre-log.1586314991.56061 [1751567.892653] Pid: 49003, comm: ll_ost_io02_015 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751567.903576] Call Trace: [1751567.906209] [] osd_trans_stop+0x265/0x8e0 [osd_ldiskfs] [1751567.913323] [] ofd_trans_stop+0x25/0x60 [ofd] [1751567.919539] [] ofd_commitrw_write+0x9d4/0x1db0 [ofd] [1751567.926393] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751567.932620] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751567.939432] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751567.946552] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751567.954495] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751567.960999] [] kthread+0xd1/0xe0 [1751567.966093] [] ret_from_fork_nospec_begin+0xe/0x21 [1751567.972735] [] 0xffffffffffffffff [1751567.977935] Pid: 49381, comm: ll_ost_io02_101 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751567.988796] Call Trace: [1751567.991427] [] bitmap_startwrite+0x1f5/0x210 [1751567.997559] [] add_stripe_bio+0x451/0x7f0 [raid456] [1751568.004311] [] raid5_make_request+0x1e4/0xca0 [raid456] [1751568.011397] [] md_handle_request+0xd0/0x150 [1751568.017439] [] md_make_request+0x79/0x190 [1751568.023318] [] generic_make_request+0x147/0x380 [1751568.029709] [] submit_bio+0x70/0x150 [1751568.035177] [] osd_submit_bio+0x1c/0x60 [osd_ldiskfs] [1751568.042101] [] osd_do_bio.isra.34+0xa45/0xb40 [osd_ldiskfs] [1751568.049567] [] osd_write_commit+0x3ec/0x8c0 [osd_ldiskfs] [1751568.056828] [] ofd_commitrw_write+0xfbe/0x1db0 [ofd] [1751568.063676] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751568.069889] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751568.076696] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751568.083830] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751568.091753] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751568.098268] [] kthread+0xd1/0xe0 [1751568.103378] [] ret_from_fork_nospec_begin+0xe/0x21 [1751568.110030] [] 0xffffffffffffffff [1751568.115238] Pid: 49348, comm: ll_ost_io03_060 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751568.126143] Call Trace: [1751568.128779] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751568.135858] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751568.143129] [] start_this_handle+0x1a1/0x430 [jbd2] [1751568.149864] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751568.156701] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751568.164311] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751568.171516] [] ofd_trans_start+0x75/0xf0 [ofd] [1751568.177814] [] ofd_commitrw_write+0xa31/0x1db0 [ofd] [1751568.184677] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751568.190893] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751568.197706] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751568.204819] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751568.212802] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751568.219302] [] kthread+0xd1/0xe0 [1751568.224423] [] ret_from_fork_nospec_begin+0xe/0x21 [1751568.231076] [] 0xffffffffffffffff [1751568.236275] Pid: 56063, comm: ll_ost_io01_074 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751568.247142] Call Trace: [1751568.249779] [] osd_trans_stop+0x265/0x8e0 [osd_ldiskfs] [1751568.256863] [] ofd_trans_stop+0x25/0x60 [ofd] [1751568.263076] [] ofd_commitrw_write+0x9d4/0x1db0 [ofd] [1751568.269902] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751568.276120] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751568.282901] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751568.290057] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751568.297942] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751568.304477] [] kthread+0xd1/0xe0 [1751568.309564] [] ret_from_fork_nospec_begin+0xe/0x21 [1751568.316247] [] 0xffffffffffffffff [1751568.321440] LNet: Service thread pid 99542 was inactive for 200.46s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751568.769099] LustreError: dumping log to /tmp/lustre-log.1586314992.48988 [1751569.281082] LNet: Service thread pid 49249 was inactive for 200.33s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751569.294122] LNet: Skipped 34 previous similar messages [1751569.299442] LustreError: dumping log to /tmp/lustre-log.1586314993.49249 [1751570.305082] LNet: Service thread pid 48882 was inactive for 200.44s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751570.318143] LNet: Skipped 18 previous similar messages [1751570.323466] LustreError: dumping log to /tmp/lustre-log.1586314994.48882 [1751571.329082] LustreError: dumping log to /tmp/lustre-log.1586314995.49255 [1751572.353078] LNet: Service thread pid 49292 was inactive for 200.41s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751572.366134] LNet: Skipped 30 previous similar messages [1751572.371498] LustreError: dumping log to /tmp/lustre-log.1586314996.49292 [1751573.377073] LustreError: dumping log to /tmp/lustre-log.1586314997.49365 [1751574.401078] LustreError: dumping log to /tmp/lustre-log.1586314998.49196 [1751575.937077] LustreError: dumping log to /tmp/lustre-log.1586314999.49111 [1751576.449075] LNet: Service thread pid 49216 was inactive for 200.35s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751576.462117] LNet: Skipped 26 previous similar messages [1751576.467438] LustreError: dumping log to /tmp/lustre-log.1586315000.49216 [1751577.985079] LustreError: dumping log to /tmp/lustre-log.1586315001.67481 [1751579.009072] LustreError: dumping log to /tmp/lustre-log.1586315002.49290 [1751579.521072] LustreError: dumping log to /tmp/lustre-log.1586315003.49204 [1751580.033073] LustreError: dumping log to /tmp/lustre-log.1586315004.49317 [1751581.057079] LustreError: dumping log to /tmp/lustre-log.1586315005.102167 [1751582.081074] LustreError: dumping log to /tmp/lustre-log.1586315006.49247 [1751583.105069] LustreError: dumping log to /tmp/lustre-log.1586315007.49135 [1751584.129072] LustreError: dumping log to /tmp/lustre-log.1586315008.49009 [1751585.153064] LNet: Service thread pid 56064 was inactive for 200.72s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751585.166114] LNet: Skipped 36 previous similar messages [1751585.171439] LustreError: dumping log to /tmp/lustre-log.1586315009.56064 [1751586.177066] LustreError: dumping log to /tmp/lustre-log.1586315010.49262 [1751587.201061] LustreError: dumping log to /tmp/lustre-log.1586315011.49236 [1751589.249061] LustreError: dumping log to /tmp/lustre-log.1586315013.49130 [1751591.297065] LustreError: dumping log to /tmp/lustre-log.1586315015.49150 [1751592.833059] LustreError: dumping log to /tmp/lustre-log.1586315016.48904 [1751593.857059] LustreError: dumping log to /tmp/lustre-log.1586315017.67500 [1751594.369059] LustreError: dumping log to /tmp/lustre-log.1586315018.50700 [1751596.417058] LustreError: dumping log to /tmp/lustre-log.1586315020.49072 [1751597.442055] LustreError: dumping log to /tmp/lustre-log.1586315021.91446 [1751599.489052] LustreError: dumping log to /tmp/lustre-log.1586315023.49306 [1751602.049060] LNet: Service thread pid 49257 was inactive for 200.27s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751602.062097] LNet: Skipped 26 previous similar messages [1751602.067412] LustreError: dumping log to /tmp/lustre-log.1586315026.49257 [1751603.073055] LustreError: dumping log to /tmp/lustre-log.1586315027.48979 [1751604.097050] LustreError: dumping log to /tmp/lustre-log.1586315028.49288 [1751605.633047] LustreError: dumping log to /tmp/lustre-log.1586315029.91682 [1751608.193046] LustreError: dumping log to /tmp/lustre-log.1586315032.48940 [1751610.753044] LustreError: dumping log to /tmp/lustre-log.1586315034.49356 [1751613.825047] LustreError: dumping log to /tmp/lustre-log.1586315037.48907 [1751619.969041] LustreError: dumping log to /tmp/lustre-log.1586315043.50850 [1751623.553031] LustreError: dumping log to /tmp/lustre-log.1586315047.49278 [1751627.137028] LustreError: dumping log to /tmp/lustre-log.1586315051.48814 [1751629.697027] LustreError: dumping log to /tmp/lustre-log.1586315053.49232 [1751630.209022] LustreError: dumping log to /tmp/lustre-log.1586315054.51672 [1751631.233029] LustreError: dumping log to /tmp/lustre-log.1586315055.49082 [1751632.769022] LustreError: dumping log to /tmp/lustre-log.1586315056.48813 [1751636.353021] LNet: Service thread pid 56058 was inactive for 200.31s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751636.366078] LNet: Skipped 29 previous similar messages [1751636.371400] LustreError: dumping log to /tmp/lustre-log.1586315060.56058 [1751637.889022] LustreError: dumping log to /tmp/lustre-log.1586315061.48313 [1751640.449017] LustreError: dumping log to /tmp/lustre-log.1586315064.56057 [1751641.473013] LustreError: dumping log to /tmp/lustre-log.1586315065.49319 [1751673.216932] LustreError: dumping log to /tmp/lustre-log.1586315097.67482 [1751675.264926] LustreError: dumping log to /tmp/lustre-log.1586315099.49083 [1751682.944910] LustreError: dumping log to /tmp/lustre-log.1586315106.49118 [1751701.376895] LNet: Service thread pid 49275 was inactive for 240.48s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751701.389928] LNet: Skipped 6 previous similar messages [1751701.395169] LustreError: dumping log to /tmp/lustre-log.1586315125.49275 [1751702.912890] LustreError: dumping log to /tmp/lustre-log.1586315126.49252 [1751713.152889] LustreError: dumping log to /tmp/lustre-log.1586315137.99543 [1751747.968856] LustreError: dumping log to /tmp/lustre-log.1586315171.49085 [1751750.016856] LustreError: dumping log to /tmp/lustre-log.1586315174.102176 [1751753.088855] LustreError: dumping log to /tmp/lustre-log.1586315177.91693 [1751755.136848] LustreError: dumping log to /tmp/lustre-log.1586315179.88758 [1751774.592829] LustreError: dumping log to /tmp/lustre-log.1586315198.48329 [1751775.616831] LustreError: dumping log to /tmp/lustre-log.1586315199.49310 [1751776.640833] LustreError: dumping log to /tmp/lustre-log.1586315200.91654 [1751779.712823] LustreError: dumping log to /tmp/lustre-log.1586315203.49250 [1751781.760820] LustreError: dumping log to /tmp/lustre-log.1586315205.49103 [1751788.929816] LustreError: dumping log to /tmp/lustre-log.1586315212.49341 [1751790.976807] LustreError: dumping log to /tmp/lustre-log.1586315214.91689 [1751826.816769] LustreError: dumping log to /tmp/lustre-log.1586315250.56065 [1751836.656754] LustreError: 49116:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586314960, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0034_UUID lock: ffff9ffccb180d80/0xf3793efb89b482ad lrc: 3/0,1 mode: --/PW res: [0x1000000401:0x85ce67:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49116 timeout: 0 lvb_type: 0 [1751836.700662] LustreError: dumping log to /tmp/lustre-log.1586315260.49116 [1751849.344739] LNet: Service thread pid 49092 was inactive for 340.80s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1751849.357775] LNet: Skipped 24 previous similar messages [1751849.363097] LustreError: dumping log to /tmp/lustre-log.1586315273.49092 [1751855.488734] LustreError: dumping log to /tmp/lustre-log.1586315279.91666 [1751861.632722] LustreError: dumping log to /tmp/lustre-log.1586315285.49124 [1751866.752724] LustreError: dumping log to /tmp/lustre-log.1586315290.48338 [1751867.776717] LNet: Service thread pid 48341 was inactive for 347.11s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1751867.793831] LNet: Skipped 4 previous similar messages [1751867.799099] Pid: 48341, comm: ll_ost_io03_002 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751867.809981] Call Trace: [1751867.812625] [] __wait_on_freeing_inode+0xb0/0xf0 [1751867.819105] [] find_inode_fast+0x71/0xb0 [1751867.824894] [] iget_locked+0x79/0x1a0 [1751867.830405] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1751867.836888] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1751867.843371] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1751867.850828] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1751867.857828] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1751867.864996] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1751867.872096] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1751867.879656] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1751867.886793] [] lu_object_find+0x16/0x20 [obdclass] [1751867.893471] [] ofd_object_find+0x35/0x100 [ofd] [1751867.899860] [] ofd_preprw_write.isra.30+0x85/0xea0 [ofd] [1751867.907059] [] ofd_preprw+0x41f/0x1240 [ofd] [1751867.913190] [] tgt_brw_write+0xd33/0x1d60 [ptlrpc] [1751867.919899] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751867.927014] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751867.934916] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751867.941451] [] kthread+0xd1/0xe0 [1751867.946553] [] ret_from_fork_nospec_begin+0xe/0x21 [1751867.953241] [] 0xffffffffffffffff [1751867.958464] LustreError: dumping log to /tmp/lustre-log.1586315291.48341 [1751871.679722] LustreError: 48958:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586314995, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0034_UUID lock: ffff9fe505520240/0xf3793efb89b4abfe lrc: 3/0,1 mode: --/PW res: [0x1000000401:0x85ce74:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48958 timeout: 0 lvb_type: 0 [1751896.448692] LNet: Service thread pid 49384 was inactive for 372.80s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1751896.465822] Pid: 49384, comm: ll_ost_io02_104 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751896.476706] Call Trace: [1751896.479370] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751896.486450] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751896.493722] [] start_this_handle+0x1a1/0x430 [jbd2] [1751896.500486] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751896.507324] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751896.514953] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751896.522144] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1751896.529007] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751896.535238] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751896.542057] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751896.549194] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751896.557112] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751896.563640] [] kthread+0xd1/0xe0 [1751896.568747] [] ret_from_fork_nospec_begin+0xe/0x21 [1751896.575410] [] 0xffffffffffffffff [1751896.580626] LustreError: dumping log to /tmp/lustre-log.1586315320.49384 [1751900.544682] LNet: Service thread pid 49352 was inactive for 373.44s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1751900.561795] Pid: 49352, comm: ll_ost_io03_062 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751900.572693] Call Trace: [1751900.575355] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751900.582443] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751900.589715] [] start_this_handle+0x1a1/0x430 [jbd2] [1751900.596486] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751900.603311] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751900.610943] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751900.618139] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1751900.625003] [] ofd_commitrw+0x47c/0xa50 [ofd] [1751900.631232] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1751900.638030] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751900.645160] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751900.653081] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751900.659593] [] kthread+0xd1/0xe0 [1751900.664705] [] ret_from_fork_nospec_begin+0xe/0x21 [1751900.671368] [] 0xffffffffffffffff [1751900.676583] LustreError: dumping log to /tmp/lustre-log.1586315324.49352 [1751901.568687] Pid: 49116, comm: ll_ost03_077 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751901.579294] Call Trace: [1751901.581940] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1751901.589086] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1751901.596473] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1751901.603232] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1751901.609725] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751901.616844] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751901.624737] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751901.631239] [] kthread+0xd1/0xe0 [1751901.636327] [] ret_from_fork_nospec_begin+0xe/0x21 [1751901.642981] [] 0xffffffffffffffff [1751901.648173] LustreError: dumping log to /tmp/lustre-log.1586315325.49116 [1751904.640687] LNet: Service thread pid 49374 was inactive for 372.62s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1751904.657834] LNet: Skipped 1 previous similar message [1751904.662987] Pid: 49374, comm: ll_ost_io02_094 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1751904.673928] Call Trace: [1751904.676585] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1751904.683669] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1751904.690963] [] start_this_handle+0x1a1/0x430 [jbd2] [1751904.697701] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1751904.704558] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1751904.712235] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1751904.719412] [] ofd_trans_start+0x75/0xf0 [ofd] [1751904.725771] [] ofd_object_punch+0x798/0xd90 [ofd] [1751904.732398] [] ofd_punch_hdl+0x4f3/0xa80 [ofd] [1751904.738702] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1751904.745853] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1751904.753753] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1751904.760300] [] kthread+0xd1/0xe0 [1751904.765455] [] ret_from_fork_nospec_begin+0xe/0x21 [1751904.772136] [] 0xffffffffffffffff [1751904.777350] LustreError: dumping log to /tmp/lustre-log.1586315328.49374 [1751909.760678] LustreError: dumping log to /tmp/lustre-log.1586315333.48934 [1751923.072664] LustreError: dumping log to /tmp/lustre-log.1586315347.48955 [1751924.096665] LustreError: dumping log to /tmp/lustre-log.1586315348.49128 [1751925.120661] LustreError: dumping log to /tmp/lustre-log.1586315349.49325 [1751928.192660] LustreError: dumping log to /tmp/lustre-log.1586315352.49281 [1751932.288650] LustreError: dumping log to /tmp/lustre-log.1586315356.102169 [1751935.360653] LustreError: dumping log to /tmp/lustre-log.1586315359.49246 [1751937.408652] LustreError: dumping log to /tmp/lustre-log.1586315361.49263 [1751938.432648] LustreError: dumping log to /tmp/lustre-log.1586315362.48890 [1751939.264654] LustreError: 49027:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586315063, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0034_UUID lock: ffffa0042e5cb180/0xf3793efb89b4fe1b lrc: 3/0,1 mode: --/PW res: [0x1000000401:0x85ce71:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49027 timeout: 0 lvb_type: 0 [1751940.480646] LustreError: dumping log to /tmp/lustre-log.1586315364.91672 [1751962.124641] Lustre: 49320:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9ffbfd7c6850 x1659645031362752/t30126767971(0) o4->159b0458-c3d3-4@10.50.17.39@o2ib2:746/0 lens 776/520 e 23 to 0 dl 1586315391 ref 2 fl Interpret:/0/0 rc 0/0 [1751963.126635] Lustre: 49316:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9ff6a961f850 x1659572575432000/t0(0) o10->f46ca3a4-70aa-4@10.50.17.40@o2ib2:747/0 lens 440/432 e 23 to 0 dl 1586315392 ref 2 fl Interpret:/0/0 rc 0/0 [1751963.154356] Lustre: 49316:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 21 previous similar messages [1751964.128628] Lustre: 49353:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9ff837552850 x1659523381250688/t0(0) o4->232937e8-bc43-4@10.49.27.31@o2ib1:748/0 lens 8792/448 e 24 to 0 dl 1586315393 ref 2 fl Interpret:/0/0 rc 0/0 [1751964.156113] Lustre: 49353:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 20 previous similar messages [1751966.132629] Lustre: 49382:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9ffb74f5d850 x1659468457672832/t0(0) o10->e7748b0d-9d27-4@10.49.28.5@o2ib1:750/0 lens 440/432 e 23 to 0 dl 1586315395 ref 2 fl Interpret:/0/0 rc 0/0 [1751966.159930] Lustre: 49382:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 31 previous similar messages [1751968.050261] Lustre: fir-OST0034: Client ef6c32ab-9321-4 (at 10.50.1.9@o2ib2) reconnecting [1751968.058618] Lustre: Skipped 11 previous similar messages [1751968.064135] Lustre: fir-OST0034: Connection restored to ef6c32ab-9321-4 (at 10.50.1.9@o2ib2) [1751968.644699] Lustre: fir-OST0034: Connection restored to fb2c1382-8f5a-4 (at 10.50.15.10@o2ib2) [1751968.653495] Lustre: Skipped 2 previous similar messages [1751969.264570] Lustre: fir-OST0034: deleting orphan objects from 0x1000000400:29597093 to 0x1000000400:29597505 [1751970.132676] Lustre: fir-OST0034: Connection restored to 203cf276-de8a-4 (at 10.49.7.13@o2ib1) [1751970.141392] Lustre: Skipped 7 previous similar messages [1751970.448626] Lustre: 91676:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffffa00b85f18850 x1659669780685952/t0(0) o4->c7633cf9-6401-4@10.50.1.43@o2ib2:754/0 lens 488/448 e 23 to 0 dl 1586315399 ref 2 fl Interpret:/0/0 rc 0/0 [1751970.475791] Lustre: 91676:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 25 previous similar messages [1751972.224607] LustreError: dumping log to /tmp/lustre-log.1586315396.49361 [1751972.505284] Lustre: fir-OST0034: Connection restored to fir-MDT0003-mdtlov_UUID (at 10.0.10.54@o2ib7) [1751972.514822] Lustre: Skipped 8 previous similar messages [1751976.320611] LustreError: dumping log to /tmp/lustre-log.1586315400.91694 [1751978.464616] Lustre: 49268:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffffa00b968f9050 x1659266293792256/t0(0) o10->15f97309-8595-4@10.50.5.49@o2ib2:7/0 lens 440/432 e 17 to 0 dl 1586315407 ref 2 fl Interpret:/0/0 rc 0/0 [1751978.491732] Lustre: 49268:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 22 previous similar messages [1751979.060933] Lustre: fir-OST0034: Client d0a45662-de66-4 (at 10.50.10.6@o2ib2) reconnecting [1751979.069389] Lustre: Skipped 26 previous similar messages [1751979.074913] Lustre: fir-OST0034: Connection restored to d0a45662-de66-4 (at 10.50.10.6@o2ib2) [1751979.083635] Lustre: Skipped 6 previous similar messages [1751986.560588] LustreError: dumping log to /tmp/lustre-log.1586315410.48958 [1751987.231000] Lustre: fir-OST0034: Connection restored to d241ea4c-b562-4 (at 10.50.7.67@o2ib2) [1751987.239707] Lustre: Skipped 8 previous similar messages [1751994.496600] Lustre: 49183:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffffa00bfde76850 x1659502839958848/t0(0) o4->c5d268c3-70a3-4@10.50.10.28@o2ib2:23/0 lens 488/448 e 12 to 0 dl 1586315423 ref 2 fl Interpret:/0/0 rc 0/0 [1751994.523797] Lustre: 49183:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 23 previous similar messages [1751998.518177] Lustre: fir-OST0034: Client 48495bd3-7f9d-4 (at 10.50.17.48@o2ib2) reconnecting [1751998.526707] Lustre: Skipped 12 previous similar messages [1752004.090685] Lustre: fir-OST0034: Connection restored to 2d94e235-5333-4 (at 10.49.31.5@o2ib1) [1752004.099394] Lustre: Skipped 8 previous similar messages [1752004.992574] LustreError: dumping log to /tmp/lustre-log.1586315428.49332 [1752007.040579] LustreError: dumping log to /tmp/lustre-log.1586315431.67499 [1752017.280571] LustreError: dumping log to /tmp/lustre-log.1586315441.49331 [1752019.328578] LustreError: dumping log to /tmp/lustre-log.1586315443.49280 [1752026.564565] Lustre: 91698:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffffa0090f4ed850 x1659445269057664/t0(0) o4->d553bc1a-5252-4@10.49.27.6@o2ib1:55/0 lens 488/448 e 7 to 0 dl 1586315455 ref 2 fl Interpret:/0/0 rc 0/0 [1752026.591557] Lustre: 91698:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 23 previous similar messages [1752031.957439] Lustre: fir-OST0034: Client d553bc1a-5252-4 (at 10.49.27.6@o2ib1) reconnecting [1752031.965884] Lustre: Skipped 14 previous similar messages [1752045.952526] LustreError: dumping log to /tmp/lustre-log.1586315469.49294 [1752050.048533] LustreError: dumping log to /tmp/lustre-log.1586315474.49371 [1752052.096540] LustreError: dumping log to /tmp/lustre-log.1586315476.49002 [1752055.025698] Lustre: fir-OST0034: Connection restored to 44afe395-037d-4 (at 10.50.10.29@o2ib2) [1752055.034488] Lustre: Skipped 10 previous similar messages [1752074.624508] LustreError: dumping log to /tmp/lustre-log.1586315498.49349 [1752078.720492] LustreError: dumping log to /tmp/lustre-log.1586315502.49042 [1752080.768496] LustreError: dumping log to /tmp/lustre-log.1586315504.49360 [1752082.816488] LustreError: dumping log to /tmp/lustre-log.1586315506.49388 [1752084.864487] LustreError: dumping log to /tmp/lustre-log.1586315508.91670 [1752088.960492] LustreError: dumping log to /tmp/lustre-log.1586315512.48878 [1752091.008489] LustreError: dumping log to /tmp/lustre-log.1586315514.49364 [1752098.832504] Lustre: 93470:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9fe84a218050 x1659416949051776/t0(0) o4->5b88c4a2-39df-4@10.50.17.45@o2ib2:127/0 lens 488/448 e 4 to 0 dl 1586315527 ref 2 fl Interpret:/0/0 rc 0/0 [1752098.859688] Lustre: 93470:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 30 previous similar messages [1752117.632462] LNet: Service thread pid 49039 was inactive for 514.84s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1752117.645527] LNet: Skipped 43 previous similar messages [1752117.650853] LustreError: dumping log to /tmp/lustre-log.1586315541.49039 [1752119.680459] LustreError: dumping log to /tmp/lustre-log.1586315543.49239 [1752121.749357] Lustre: fir-OST0034: Client 54e43b9d-9d40-4 (at 10.50.4.3@o2ib2) reconnecting [1752121.757715] Lustre: Skipped 7 previous similar messages [1752121.763148] Lustre: fir-OST0034: Connection restored to 54e43b9d-9d40-4 (at 10.50.4.3@o2ib2) [1752121.771784] Lustre: Skipped 6 previous similar messages [1752123.776450] LustreError: dumping log to /tmp/lustre-log.1586315547.48864 [1752125.824576] LustreError: dumping log to /tmp/lustre-log.1586315549.49289 [1752144.256421] LustreError: dumping log to /tmp/lustre-log.1586315568.49208 [1752150.400421] LustreError: dumping log to /tmp/lustre-log.1586315574.49010 [1752152.448420] LustreError: dumping log to /tmp/lustre-log.1586315576.49282 [1752154.496410] LustreError: dumping log to /tmp/lustre-log.1586315578.48998 [1752158.592413] LustreError: dumping log to /tmp/lustre-log.1586315582.49107 [1752160.640405] LustreError: dumping log to /tmp/lustre-log.1586315584.49408 [1752189.312372] LNet: Service thread pid 49164 was inactive for 565.28s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1752189.329512] Pid: 49164, comm: ll_ost03_087 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752189.340141] Call Trace: [1752189.342802] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752189.349888] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752189.357144] [] start_this_handle+0x1a1/0x430 [jbd2] [1752189.363898] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752189.370722] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752189.378332] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1752189.385508] [] dqget+0x3fa/0x450 [1752189.390611] [] dquot_get_dqblk+0x14/0x1f0 [1752189.396480] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1752189.404193] [] lquotactl_slv+0x27d/0xa00 [lquota] [1752189.410756] [] ofd_quotactl+0x13c/0x380 [ofd] [1752189.416983] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752189.424110] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752189.432012] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752189.438516] [] kthread+0xd1/0xe0 [1752189.443614] [] ret_from_fork_nospec_begin+0xe/0x21 [1752189.450265] [] 0xffffffffffffffff [1752189.455478] LustreError: dumping log to /tmp/lustre-log.1586315613.49164 [1752195.456370] Pid: 91684, comm: ll_ost_io01_089 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752195.467237] Call Trace: [1752195.469898] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752195.476983] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752195.484236] [] start_this_handle+0x1a1/0x430 [jbd2] [1752195.490970] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752195.497790] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752195.505400] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752195.512597] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752195.519421] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752195.525635] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752195.532424] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752195.539537] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752195.547434] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752195.553941] [] kthread+0xd1/0xe0 [1752195.559029] [] ret_from_fork_nospec_begin+0xe/0x21 [1752195.565677] [] 0xffffffffffffffff [1752195.570861] LustreError: dumping log to /tmp/lustre-log.1586315619.91684 [1752199.552375] Pid: 48951, comm: ll_ost_io01_009 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752199.563257] Call Trace: [1752199.565928] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752199.573054] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752199.580328] [] start_this_handle+0x1a1/0x430 [jbd2] [1752199.587086] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752199.593925] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752199.601578] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752199.608780] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752199.615656] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752199.621894] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752199.628722] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752199.635877] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752199.643811] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752199.650344] [] kthread+0xd1/0xe0 [1752199.655466] [] ret_from_fork_nospec_begin+0xe/0x21 [1752199.662132] [] 0xffffffffffffffff [1752199.667365] LustreError: dumping log to /tmp/lustre-log.1586315623.48951 [1752199.674702] Pid: 49346, comm: ll_ost_io02_072 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752199.685595] Call Trace: [1752199.688245] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752199.695368] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752199.702646] [] start_this_handle+0x1a1/0x430 [jbd2] [1752199.709416] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752199.716255] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752199.723893] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752199.731076] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752199.737920] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752199.744137] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752199.750929] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752199.758055] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752199.765997] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752199.772520] [] kthread+0xd1/0xe0 [1752199.777634] [] ret_from_fork_nospec_begin+0xe/0x21 [1752199.784317] [] 0xffffffffffffffff [1752203.648363] Pid: 49027, comm: ll_ost03_056 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752203.658970] Call Trace: [1752203.661616] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1752203.668751] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1752203.676133] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1752203.682898] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1752203.689375] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752203.696494] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752203.704397] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752203.710897] [] kthread+0xd1/0xe0 [1752203.715998] [] ret_from_fork_nospec_begin+0xe/0x21 [1752203.722666] [] 0xffffffffffffffff [1752203.727874] LustreError: dumping log to /tmp/lustre-log.1586315627.49027 [1752226.176343] LustreError: dumping log to /tmp/lustre-log.1586315650.51675 [1752228.224331] LustreError: dumping log to /tmp/lustre-log.1586315652.49357 [1752230.068356] Lustre: 93575:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9fdbfd133850 x1659279725146816/t0(0) o4->fec02622-e2c9-4@10.50.10.10@o2ib2:259/0 lens 2472/448 e 2 to 0 dl 1586315659 ref 2 fl Interpret:/0/0 rc 0/0 [1752230.095607] Lustre: 93575:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 62 previous similar messages [1752232.320342] LustreError: dumping log to /tmp/lustre-log.1586315656.51032 [1752258.846867] Lustre: fir-OST0034: Client 0f2d3178-05c2-4 (at 10.49.0.64@o2ib1) reconnecting [1752258.855307] Lustre: Skipped 13 previous similar messages [1752258.860820] Lustre: fir-OST0034: Connection restored to 0f2d3178-05c2-4 (at 10.49.0.64@o2ib1) [1752258.869521] Lustre: Skipped 13 previous similar messages [1752273.280280] LustreError: dumping log to /tmp/lustre-log.1586315697.49146 [1752299.904266] LustreError: dumping log to /tmp/lustre-log.1586315723.49330 [1752301.952274] LustreError: dumping log to /tmp/lustre-log.1586315725.91679 [1752306.048255] LustreError: dumping log to /tmp/lustre-log.1586315730.49076 [1752310.144246] LustreError: dumping log to /tmp/lustre-log.1586315734.48963 [1752318.336237] LustreError: dumping log to /tmp/lustre-log.1586315742.49369 [1752328.576220] LustreError: dumping log to /tmp/lustre-log.1586315752.49437 [1752347.008207] LustreError: dumping log to /tmp/lustre-log.1586315770.48914 [1752357.248189] LustreError: dumping log to /tmp/lustre-log.1586315781.48876 [1752359.296183] LustreError: dumping log to /tmp/lustre-log.1586315783.48817 [1752369.536172] LustreError: dumping log to /tmp/lustre-log.1586315793.49415 [1752375.680179] LustreError: dumping log to /tmp/lustre-log.1586315799.49007 [1752377.728179] LustreError: dumping log to /tmp/lustre-log.1586315801.49203 [1752379.776181] LustreError: dumping log to /tmp/lustre-log.1586315803.67600 [1752379.931173] LustreError: 49226:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586315503, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0032_UUID lock: ffff9fe35f3e9200/0xf3793efb89b6d9d5 lrc: 3/0,1 mode: --/PW res: [0xf80000401:0x1c07dd5:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49226 timeout: 0 lvb_type: 0 [1752381.824166] LustreError: dumping log to /tmp/lustre-log.1586315805.67599 [1752383.872165] LustreError: dumping log to /tmp/lustre-log.1586315807.51667 [1752412.544136] LustreError: dumping log to /tmp/lustre-log.1586315836.49406 [1752414.592124] LustreError: dumping log to /tmp/lustre-log.1586315838.50948 [1752426.880116] LustreError: dumping log to /tmp/lustre-log.1586315850.48336 [1752449.408088] LustreError: dumping log to /tmp/lustre-log.1586315873.49069 [1752455.552075] LustreError: dumping log to /tmp/lustre-log.1586315879.49243 [1752457.600080] LustreError: dumping log to /tmp/lustre-log.1586315881.36528 [1752463.744066] LustreError: dumping log to /tmp/lustre-log.1586315887.91691 [1752465.792062] LustreError: dumping log to /tmp/lustre-log.1586315889.48980 [1752467.840062] LustreError: dumping log to /tmp/lustre-log.1586315891.49152 [1752486.976066] Lustre: 49376:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9ffb7723a050 x1659154931744384/t0(0) o4->44afe395-037d-4@10.50.10.29@o2ib2:515/0 lens 1528/448 e 2 to 0 dl 1586315915 ref 2 fl Interpret:/0/0 rc 0/0 [1752487.003328] Lustre: 49376:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 66 previous similar messages [1752500.608027] LNet: Service thread pid 51679 was inactive for 772.43s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1752500.625134] LNet: Skipped 4 previous similar messages [1752500.630368] Pid: 51679, comm: ll_ost_io03_068 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752500.641233] Call Trace: [1752500.643895] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752500.650983] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752500.658262] [] start_this_handle+0x1a1/0x430 [jbd2] [1752500.665010] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752500.671854] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752500.679483] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752500.686688] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752500.693522] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752500.699764] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752500.706564] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752500.713682] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752500.721598] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752500.728096] [] kthread+0xd1/0xe0 [1752500.733197] [] ret_from_fork_nospec_begin+0xe/0x21 [1752500.739864] [] 0xffffffffffffffff [1752500.745064] LustreError: dumping log to /tmp/lustre-log.1586315924.51679 [1752500.752516] Pid: 48910, comm: ll_ost_io02_007 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752500.763410] Call Trace: [1752500.766047] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752500.773127] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752500.780397] [] start_this_handle+0x1a1/0x430 [jbd2] [1752500.787147] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752500.793984] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752500.801590] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752500.808772] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752500.815595] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752500.821847] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752500.828621] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752500.835759] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752500.843648] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752500.850179] [] kthread+0xd1/0xe0 [1752500.855276] [] ret_from_fork_nospec_begin+0xe/0x21 [1752500.861941] [] 0xffffffffffffffff [1752500.915023] LustreError: 48849:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586315624, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0036_UUID lock: ffff9ff2b51c9b00/0xf3793efb89b7dc57 lrc: 3/0,1 mode: --/PW res: [0x1080000402:0x10b996e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48849 timeout: 0 lvb_type: 0 [1752504.704025] Pid: 49212, comm: ll_ost_io00_010 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752504.714923] Call Trace: [1752504.717591] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752504.724735] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752504.732043] [] start_this_handle+0x1a1/0x430 [jbd2] [1752504.738827] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752504.745688] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752504.753371] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752504.760582] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752504.767468] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752504.773720] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752504.780581] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752504.787735] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752504.795688] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752504.802226] [] kthread+0xd1/0xe0 [1752504.807391] [] ret_from_fork_nospec_begin+0xe/0x21 [1752504.814085] [] 0xffffffffffffffff [1752504.819346] LustreError: dumping log to /tmp/lustre-log.1586315928.49212 [1752525.184000] Pid: 91688, comm: ll_ost_io01_091 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752525.194866] Call Trace: [1752525.197536] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752525.204632] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752525.211889] [] start_this_handle+0x1a1/0x430 [jbd2] [1752525.218624] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752525.225445] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752525.233060] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752525.240239] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752525.247061] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752525.253275] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752525.260060] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752525.267210] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752525.275154] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752525.281667] [] kthread+0xd1/0xe0 [1752525.286797] [] ret_from_fork_nospec_begin+0xe/0x21 [1752525.293449] [] 0xffffffffffffffff [1752525.298673] LustreError: dumping log to /tmp/lustre-log.1586315949.91688 [1752525.306175] Pid: 49129, comm: ll_ost_io00_007 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752525.317089] Call Trace: [1752525.319747] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752525.326828] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752525.334083] [] start_this_handle+0x1a1/0x430 [jbd2] [1752525.340814] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752525.347640] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752525.355253] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752525.362421] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752525.369255] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752525.375473] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752525.382256] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752525.389366] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752525.397252] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752525.403752] [] kthread+0xd1/0xe0 [1752525.408840] [] ret_from_fork_nospec_begin+0xe/0x21 [1752525.415486] [] 0xffffffffffffffff [1752538.001721] Lustre: fir-OST0034: Client 2a442b89-d680-4 (at 10.50.6.7@o2ib2) reconnecting [1752538.010082] Lustre: Skipped 18 previous similar messages [1752538.015594] Lustre: fir-OST0034: Connection restored to 2a442b89-d680-4 (at 10.50.6.7@o2ib2) [1752538.024230] Lustre: Skipped 18 previous similar messages [1752540.215008] LustreError: 49046:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586315664, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0030_UUID lock: ffff9fef408e2ac0/0xf3793efb89b7e9e0 lrc: 3/0,1 mode: --/PW res: [0xf00000402:0x10b7292:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49046 timeout: 0 lvb_type: 0 [1752574.335940] LustreError: dumping log to /tmp/lustre-log.1586315998.49286 [1752580.479936] LustreError: dumping log to /tmp/lustre-log.1586316004.49272 [1752590.279938] LustreError: 48978:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586315714, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0036_UUID lock: ffff9ff9a4dbd7c0/0xf3793efb89b7ed05 lrc: 3/0,1 mode: --/PW res: [0x1080000402:0x10b996f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48978 timeout: 0 lvb_type: 0 [1752598.911929] LustreError: dumping log to /tmp/lustre-log.1586316022.49399 [1752600.959932] LustreError: dumping log to /tmp/lustre-log.1586316024.49351 [1752615.295902] LustreError: dumping log to /tmp/lustre-log.1586316039.68485 [1752650.111870] LNet: Service thread pid 51677 was inactive for 873.92s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1752650.124899] LNet: Skipped 53 previous similar messages [1752650.130224] LustreError: dumping log to /tmp/lustre-log.1586316074.51677 [1752676.515841] LustreError: 49225:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586315800, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0036_UUID lock: ffff9fe91f2b60c0/0xf3793efb89b7f310 lrc: 3/0,1 mode: --/PW res: [0x1080000400:0x1c1af8f:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49225 timeout: 0 lvb_type: 0 [1752678.783848] LustreError: dumping log to /tmp/lustre-log.1586316102.49228 [1752689.023829] LustreError: dumping log to /tmp/lustre-log.1586316113.49217 [1752693.119837] LustreError: dumping log to /tmp/lustre-log.1586316117.102177 [1752697.215831] LustreError: dumping log to /tmp/lustre-log.1586316121.49161 [1752725.887793] LustreError: dumping log to /tmp/lustre-log.1586316149.49260 [1752729.983785] LustreError: dumping log to /tmp/lustre-log.1586316153.49167 [1752754.559768] LustreError: dumping log to /tmp/lustre-log.1586316178.49041 [1752799.615717] LustreError: dumping log to /tmp/lustre-log.1586316223.102170 [1752801.663710] LNet: Service thread pid 49235 was inactive for 973.90s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1752801.680968] LNet: Skipped 4 previous similar messages [1752801.686219] Pid: 49235, comm: ll_ost_io02_034 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752801.697256] Call Trace: [1752801.700132] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752801.707650] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752801.715318] [] start_this_handle+0x1a1/0x430 [jbd2] [1752801.722463] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752801.729744] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752801.737765] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752801.745384] [] ofd_trans_start+0x75/0xf0 [ofd] [1752801.752066] [] ofd_commitrw_write+0xa31/0x1db0 [ofd] [1752801.759230] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752801.765816] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752801.772909] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752801.780237] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752801.788418] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752801.795224] [] kthread+0xd1/0xe0 [1752801.800598] [] ret_from_fork_nospec_begin+0xe/0x21 [1752801.807547] [] 0xffffffffffffffff [1752801.813054] LustreError: dumping log to /tmp/lustre-log.1586316225.49235 [1752805.759711] Pid: 49240, comm: ll_ost_io01_035 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752805.770577] Call Trace: [1752805.773270] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752805.780349] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752805.787604] [] start_this_handle+0x1a1/0x430 [jbd2] [1752805.794338] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752805.801157] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752805.808767] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752805.815959] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752805.822782] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752805.828995] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752805.835814] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752805.842948] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752805.850875] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752805.857402] [] kthread+0xd1/0xe0 [1752805.862492] [] ret_from_fork_nospec_begin+0xe/0x21 [1752805.869139] [] 0xffffffffffffffff [1752805.874356] LustreError: dumping log to /tmp/lustre-log.1586316229.49240 [1752807.807708] Pid: 48933, comm: ll_ost_io02_009 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752807.818600] Call Trace: [1752807.821284] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752807.828434] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752807.835740] [] start_this_handle+0x1a1/0x430 [jbd2] [1752807.842544] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752807.849420] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752807.857115] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752807.864344] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752807.871231] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752807.877501] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752807.884356] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752807.891522] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752807.899487] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752807.906048] [] kthread+0xd1/0xe0 [1752807.911203] [] ret_from_fork_nospec_begin+0xe/0x21 [1752807.917902] [] 0xffffffffffffffff [1752807.923171] LustreError: dumping log to /tmp/lustre-log.1586316231.48933 [1752828.287686] Pid: 49202, comm: ll_ost_io01_025 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752828.298549] Call Trace: [1752828.301210] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752828.308288] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752828.315561] [] start_this_handle+0x1a1/0x430 [jbd2] [1752828.322293] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752828.329132] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752828.336740] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752828.343933] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1752828.350771] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752828.357002] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752828.363790] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752828.370920] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752828.378809] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752828.385325] [] kthread+0xd1/0xe0 [1752828.390415] [] ret_from_fork_nospec_begin+0xe/0x21 [1752828.397075] [] 0xffffffffffffffff [1752828.402263] LustreError: dumping log to /tmp/lustre-log.1586316252.49202 [1752836.260581] Lustre: fir-OST0032: deleting orphan objects from 0xf80000401:29392355 to 0xf80000401:29392993 [1752840.575671] Pid: 49344, comm: ll_ost_io03_058 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1752840.586538] Call Trace: [1752840.589204] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1752840.596283] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1752840.603539] [] start_this_handle+0x1a1/0x430 [jbd2] [1752840.610272] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1752840.617094] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1752840.624704] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1752840.631871] [] ofd_trans_start+0x75/0xf0 [ofd] [1752840.638171] [] ofd_commitrw_write+0xa31/0x1db0 [ofd] [1752840.644990] [] ofd_commitrw+0x47c/0xa50 [ofd] [1752840.651205] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1752840.657992] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1752840.665105] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1752840.673001] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1752840.679511] [] kthread+0xd1/0xe0 [1752840.684596] [] ret_from_fork_nospec_begin+0xe/0x21 [1752840.691245] [] 0xffffffffffffffff [1752840.696430] LustreError: dumping log to /tmp/lustre-log.1586316264.49344 [1752877.439636] LustreError: dumping log to /tmp/lustre-log.1586316301.49322 [1752879.487642] LustreError: dumping log to /tmp/lustre-log.1586316303.49071 [1752879.660642] LustreError: 49412:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586316003, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0032_UUID lock: ffff9fde42d62d00/0xf3793efb89ba2c8d lrc: 3/0,1 mode: --/PW res: [0xf80000401:0x1c07dde:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49412 timeout: 0 lvb_type: 0 [1752879.704483] LustreError: 49412:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 2 previous similar messages [1752906.111596] LustreError: dumping log to /tmp/lustre-log.1586316330.49323 [1752910.207595] LustreError: dumping log to /tmp/lustre-log.1586316334.49251 [1752956.473334] Lustre: fir-OST0036: deleting orphan objects from 0x1080000402:17537399 to 0x1080000402:17537633 [1752975.743523] LustreError: dumping log to /tmp/lustre-log.1586316399.91669 [1752979.839523] LustreError: dumping log to /tmp/lustre-log.1586316403.49234 [1752983.935530] LustreError: dumping log to /tmp/lustre-log.1586316407.48337 [1752996.238292] Lustre: fir-OST0030: deleting orphan objects from 0xf00000402:17527451 to 0xf00000402:17527713 [1752999.091519] Lustre: 93405:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9feb9da89050 x1663062070346624/t0(0) o3->9b444a4b-8a4b-4@10.50.14.3@o2ib2:273/0 lens 488/0 e 1 to 0 dl 1586316428 ref 2 fl New:/0/ffffffff rc 0/-1 [1752999.118601] Lustre: 93405:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 824 previous similar messages [1753028.991470] LustreError: dumping log to /tmp/lustre-log.1586316452.49358 [1753032.595476] LustreError: 48893:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586316156, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0032_UUID lock: ffff9fee8b041440/0xf3793efb89bab7c2 lrc: 3/0,1 mode: --/PW res: [0xf80000402:0x84d0e0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48893 timeout: 0 lvb_type: 0 [1753032.639191] LustreError: 48893:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 2 previous similar messages [1753053.589489] Lustre: fir-OST0032: Client 7a4b5ab1-0a05-4 (at 10.50.10.40@o2ib2) reconnecting [1753053.598014] Lustre: Skipped 477 previous similar messages [1753053.603614] Lustre: fir-OST0032: Connection restored to 7a4b5ab1-0a05-4 (at 10.50.10.40@o2ib2) [1753053.612441] Lustre: Skipped 477 previous similar messages [1753061.759443] LustreError: dumping log to /tmp/lustre-log.1586316485.49267 [1753131.391394] LNet: Service thread pid 49372 was inactive for 1194.43s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1753131.408676] LNet: Skipped 4 previous similar messages [1753131.413920] Pid: 49372, comm: ll_ost_io02_092 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753131.424830] Call Trace: [1753131.427497] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753131.434625] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753131.441913] [] start_this_handle+0x1a1/0x430 [jbd2] [1753131.448698] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753131.455535] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753131.463204] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1753131.470400] [] ofd_trans_start+0x75/0xf0 [ofd] [1753131.476752] [] ofd_commitrw_write+0xa31/0x1db0 [ofd] [1753131.483601] [] ofd_commitrw+0x47c/0xa50 [ofd] [1753131.489870] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1753131.496699] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753131.503892] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753131.511841] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753131.518428] [] kthread+0xd1/0xe0 [1753131.523548] [] ret_from_fork_nospec_begin+0xe/0x21 [1753131.530217] [] 0xffffffffffffffff [1753131.535437] LustreError: dumping log to /tmp/lustre-log.1586316555.49372 [1753131.543181] Pid: 93407, comm: ll_ost_io01_098 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753131.554146] Call Trace: [1753131.556827] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753131.564055] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753131.571463] [] start_this_handle+0x1a1/0x430 [jbd2] [1753131.578361] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753131.585343] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753131.593106] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1753131.600463] [] ofd_trans_start+0x75/0xf0 [ofd] [1753131.606903] [] ofd_object_punch+0x798/0xd90 [ofd] [1753131.613607] [] ofd_punch_hdl+0x4f3/0xa80 [ofd] [1753131.620074] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753131.627361] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753131.635386] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753131.642049] [] kthread+0xd1/0xe0 [1753131.647234] [] ret_from_fork_nospec_begin+0xe/0x21 [1753131.653975] [] 0xffffffffffffffff [1753131.659282] Pid: 49338, comm: ll_ost_io02_070 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753131.670160] Call Trace: [1753131.672892] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753131.680099] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753131.687484] [] start_this_handle+0x1a1/0x430 [jbd2] [1753131.694365] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753131.701351] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753131.709080] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1753131.716433] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1753131.723420] [] ofd_commitrw+0x47c/0xa50 [ofd] [1753131.729794] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1753131.736781] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753131.744118] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753131.752183] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753131.758832] [] kthread+0xd1/0xe0 [1753131.764058] [] ret_from_fork_nospec_begin+0xe/0x21 [1753131.770853] [] 0xffffffffffffffff [1753132.468326] Lustre: fir-OST0036: deleting orphan objects from 0x1080000400:29470620 to 0x1080000400:29471361 [1753155.588955] Lustre: fir-OST003a: deleting orphan objects from 0x1180000401:8854837 to 0x1180000401:8854945 [1753160.063364] Pid: 49175, comm: ll_ost02_089 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753160.074004] Call Trace: [1753160.076689] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753160.083825] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753160.091118] [] start_this_handle+0x1a1/0x430 [jbd2] [1753160.097869] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753160.104739] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753160.112386] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1753160.119619] [] ofd_precreate_objects+0xa57/0x1e20 [ofd] [1753160.126782] [] ofd_create_hdl+0x474/0x20e0 [ofd] [1753160.133342] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753160.140615] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753160.148591] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753160.155173] [] kthread+0xd1/0xe0 [1753160.160331] [] ret_from_fork_nospec_begin+0xe/0x21 [1753160.167071] [] 0xffffffffffffffff [1753160.172375] LustreError: dumping log to /tmp/lustre-log.1586316584.49175 [1753172.351386] Pid: 48339, comm: ll_ost_io03_000 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753172.362291] Call Trace: [1753172.364956] [] __lock_page+0x74/0x90 [1753172.370410] [] __find_lock_page+0x54/0x70 [1753172.376295] [] find_or_create_page+0x34/0xa0 [1753172.382472] [] osd_bufs_get+0x413/0x870 [osd_ldiskfs] [1753172.389417] [] ofd_preprw_write.isra.30+0x476/0xea0 [ofd] [1753172.396707] [] ofd_preprw+0x41f/0x1240 [ofd] [1753172.402870] [] tgt_brw_write+0xd33/0x1d60 [ptlrpc] [1753172.409603] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753172.416781] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753172.424698] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753172.431235] [] kthread+0xd1/0xe0 [1753172.436350] [] ret_from_fork_nospec_begin+0xe/0x21 [1753172.443029] [] 0xffffffffffffffff [1753172.448266] LustreError: dumping log to /tmp/lustre-log.1586316596.48339 [1753176.447347] LustreError: dumping log to /tmp/lustre-log.1586316600.51673 [1753180.543349] LustreError: dumping log to /tmp/lustre-log.1586316604.93469 [1753184.639339] LustreError: dumping log to /tmp/lustre-log.1586316608.91676 [1753188.735332] LustreError: dumping log to /tmp/lustre-log.1586316612.67483 [1753192.831325] LustreError: dumping log to /tmp/lustre-log.1586316616.51030 [1753201.023318] LustreError: dumping log to /tmp/lustre-log.1586316625.102171 [1753205.119315] LustreError: dumping log to /tmp/lustre-log.1586316629.96830 [1753209.215318] LustreError: dumping log to /tmp/lustre-log.1586316633.49123 [1753217.407309] LustreError: dumping log to /tmp/lustre-log.1586316641.49132 [1753225.599296] LustreError: dumping log to /tmp/lustre-log.1586316649.49274 [1753233.791286] LustreError: dumping log to /tmp/lustre-log.1586316657.93514 [1753237.887282] LustreError: dumping log to /tmp/lustre-log.1586316661.49172 [1753241.941175] Lustre: fir-OST003a: deleting orphan objects from 0x1180000400:29555819 to 0x1180000400:29556529 [1753241.983287] LustreError: dumping log to /tmp/lustre-log.1586316665.48954 [1753246.079273] LustreError: dumping log to /tmp/lustre-log.1586316670.93516 [1753250.175273] LNet: Service thread pid 48973 was inactive for 1202.05s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1753250.188397] LNet: Skipped 132 previous similar messages [1753250.193806] LustreError: dumping log to /tmp/lustre-log.1586316674.48973 [1753254.271276] LustreError: dumping log to /tmp/lustre-log.1586316678.49224 [1753258.367260] LustreError: dumping log to /tmp/lustre-log.1586316682.93493 [1753262.463266] LustreError: dumping log to /tmp/lustre-log.1586316686.93527 [1753282.943247] LustreError: dumping log to /tmp/lustre-log.1586316706.49226 [1753287.039239] LustreError: dumping log to /tmp/lustre-log.1586316711.93526 [1753291.135233] LustreError: dumping log to /tmp/lustre-log.1586316715.93529 [1753295.231233] LustreError: dumping log to /tmp/lustre-log.1586316719.49058 [1753303.423224] LustreError: dumping log to /tmp/lustre-log.1586316727.112660 [1753307.519229] LustreError: dumping log to /tmp/lustre-log.1586316731.49414 [1753311.973217] LustreError: 49050:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586316435, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0030_UUID lock: ffff9feeb864d580/0xf3793efb89bbae4a lrc: 3/0,1 mode: --/PW res: [0x572c86a:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49050 timeout: 0 lvb_type: 0 [1753312.016344] LustreError: 49050:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 5 previous similar messages [1753323.903200] LustreError: dumping log to /tmp/lustre-log.1586316747.93572 [1753332.095202] LustreError: dumping log to /tmp/lustre-log.1586316756.93550 [1753336.191206] LustreError: dumping log to /tmp/lustre-log.1586316760.93569 [1753344.383212] LustreError: dumping log to /tmp/lustre-log.1586316768.93567 [1753348.479205] LustreError: dumping log to /tmp/lustre-log.1586316772.48297 [1753352.575215] LustreError: dumping log to /tmp/lustre-log.1586316776.49153 [1753356.671220] LustreError: dumping log to /tmp/lustre-log.1586316780.48889 [1753364.586112] Lustre: fir-OST0030: deleting orphan objects from 0x0:91408493 to 0x0:91408577 [1753381.247183] LustreError: dumping log to /tmp/lustre-log.1586316805.49291 [1753393.535179] LustreError: dumping log to /tmp/lustre-log.1586316817.93570 [1753401.727188] LustreError: dumping log to /tmp/lustre-log.1586316825.48849 [1753422.207148] LustreError: dumping log to /tmp/lustre-log.1586316846.48966 [1753442.687126] LNet: Service thread pid 49046 was inactive for 1202.47s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1753442.704321] LNet: Skipped 4 previous similar messages [1753442.709557] Pid: 49046, comm: ll_ost02_060 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753442.720164] Call Trace: [1753442.722810] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1753442.729928] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1753442.737312] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1753442.744046] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1753442.750520] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753442.757645] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753442.765548] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753442.772050] [] kthread+0xd1/0xe0 [1753442.777151] [] ret_from_fork_nospec_begin+0xe/0x21 [1753442.783801] [] 0xffffffffffffffff [1753442.789010] LustreError: dumping log to /tmp/lustre-log.1586316866.49046 [1753442.796542] Pid: 48968, comm: ll_ost01_037 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753442.807180] Call Trace: [1753442.809819] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753442.816899] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753442.824168] [] start_this_handle+0x1a1/0x430 [jbd2] [1753442.830902] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753442.837740] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753442.845349] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1753442.852541] [] dqget+0x3fa/0x450 [1753442.857631] [] dquot_get_dqblk+0x14/0x1f0 [1753442.863508] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1753442.871239] [] lquotactl_slv+0x27d/0xa00 [lquota] [1753442.877799] [] ofd_quotactl+0x13c/0x380 [ofd] [1753442.884011] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753442.891128] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753442.899013] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753442.905513] [] kthread+0xd1/0xe0 [1753442.910592] [] ret_from_fork_nospec_begin+0xe/0x21 [1753442.917230] [] 0xffffffffffffffff [1753444.889620] Lustre: fir-OST003a: deleting orphan objects from 0x1180000402:17593756 to 0x1180000402:17593953 [1753446.783123] Pid: 93575, comm: ll_ost_io00_090 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753446.794021] Call Trace: [1753446.796668] [] __lock_page+0x74/0x90 [1753446.802137] [] __find_lock_page+0x54/0x70 [1753446.808049] [] find_or_create_page+0x34/0xa0 [1753446.814185] [] osd_bufs_get+0x413/0x870 [osd_ldiskfs] [1753446.821102] [] ofd_preprw_write.isra.30+0x476/0xea0 [ofd] [1753446.828440] [] ofd_preprw+0x41f/0x1240 [ofd] [1753446.834606] [] tgt_brw_write+0xd33/0x1d60 [ptlrpc] [1753446.841339] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753446.848479] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753446.856455] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753446.862996] [] kthread+0xd1/0xe0 [1753446.868101] [] ret_from_fork_nospec_begin+0xe/0x21 [1753446.874808] [] 0xffffffffffffffff [1753446.880093] LustreError: dumping log to /tmp/lustre-log.1586316870.93575 [1753450.879131] Pid: 48975, comm: ll_ost02_043 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753450.889733] Call Trace: [1753450.892394] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753450.899500] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753450.906753] [] start_this_handle+0x1a1/0x430 [jbd2] [1753450.913510] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753450.920350] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753450.927984] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1753450.935180] [] dqget+0x3fa/0x450 [1753450.940287] [] dquot_get_dqblk+0x14/0x1f0 [1753450.946144] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1753450.953874] [] lquotactl_slv+0x27d/0xa00 [lquota] [1753450.960435] [] ofd_quotactl+0x13c/0x380 [ofd] [1753450.966679] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753450.973816] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753450.981735] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753450.988237] [] kthread+0xd1/0xe0 [1753450.993325] [] ret_from_fork_nospec_begin+0xe/0x21 [1753450.999972] [] 0xffffffffffffffff [1753451.005184] LustreError: dumping log to /tmp/lustre-log.1586316874.48975 [1753454.975124] Pid: 49213, comm: ll_ost02_098 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753454.985733] Call Trace: [1753454.988391] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753454.995474] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753455.002756] [] start_this_handle+0x1a1/0x430 [jbd2] [1753455.009487] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753455.016309] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753455.023918] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1753455.031093] [] dqget+0x3fa/0x450 [1753455.036190] [] dquot_get_dqblk+0x14/0x1f0 [1753455.042051] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1753455.049753] [] lquotactl_slv+0x27d/0xa00 [lquota] [1753455.056315] [] ofd_quotactl+0x13c/0x380 [ofd] [1753455.062530] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753455.069660] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753455.077548] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753455.084049] [] kthread+0xd1/0xe0 [1753455.089136] [] ret_from_fork_nospec_begin+0xe/0x21 [1753455.095782] [] 0xffffffffffffffff [1753455.100968] LustreError: dumping log to /tmp/lustre-log.1586316879.49213 [1753459.071119] LustreError: dumping log to /tmp/lustre-log.1586316883.93573 [1753467.263107] LustreError: dumping log to /tmp/lustre-log.1586316891.51678 [1753475.455098] LustreError: dumping log to /tmp/lustre-log.1586316899.48939 [1753483.647092] LustreError: dumping log to /tmp/lustre-log.1586316907.48628 [1753488.557595] Lustre: fir-OST0032: deleting orphan objects from 0xf80000402:8704237 to 0xf80000402:8704289 [1753490.097545] Lustre: fir-OST0032: deleting orphan objects from 0xf80000400:17419529 to 0xf80000400:17419809 [1753491.839082] LustreError: dumping log to /tmp/lustre-log.1586316915.48978 [1753495.935074] LustreError: dumping log to /tmp/lustre-log.1586316919.48902 [1753508.223064] LustreError: dumping log to /tmp/lustre-log.1586316932.49053 [1753512.319067] LustreError: dumping log to /tmp/lustre-log.1586316936.48947 [1753516.415071] LustreError: dumping log to /tmp/lustre-log.1586316940.48992 [1753553.279036] LustreError: dumping log to /tmp/lustre-log.1586316977.93664 [1753561.471028] LustreError: dumping log to /tmp/lustre-log.1586316985.93609 [1753573.759020] LustreError: dumping log to /tmp/lustre-log.1586316997.49061 [1753577.855009] LustreError: dumping log to /tmp/lustre-log.1586317001.48880 [1753581.951005] LustreError: dumping log to /tmp/lustre-log.1586317005.49131 [1753586.047009] LustreError: dumping log to /tmp/lustre-log.1586317010.48956 [1753592.439697] Lustre: fir-OST0032: deleting orphan objects from 0xf80000401:29393003 to 0xf80000401:29393025 [1753600.463014] Lustre: 49376:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9ff89c766050 x1659465725515584/t0(0) o3->a76a4cf2-59c5-4@10.50.4.10@o2ib2:119/0 lens 488/0 e 0 to 0 dl 1586317029 ref 2 fl New:/2/ffffffff rc 0/-1 [1753600.490151] Lustre: 49376:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1300 previous similar messages [1753602.430985] LustreError: dumping log to /tmp/lustre-log.1586317026.49163 [1753639.294941] LustreError: dumping log to /tmp/lustre-log.1586317063.93713 [1753651.582937] LustreError: dumping log to /tmp/lustre-log.1586317075.93710 [1753655.260696] Lustre: fir-OST0030: Client fe28d04b-d655-4 (at 10.50.2.47@o2ib2) reconnecting [1753655.269140] Lustre: Skipped 621 previous similar messages [1753655.274748] Lustre: fir-OST0030: Connection restored to 0c708472-c1db-4 (at 10.50.2.47@o2ib2) [1753655.283447] Lustre: Skipped 620 previous similar messages [1753655.678920] LustreError: dumping log to /tmp/lustre-log.1586317079.48286 [1753672.062903] LustreError: dumping log to /tmp/lustre-log.1586317096.48831 [1753680.254900] LustreError: dumping log to /tmp/lustre-log.1586317104.49122 [1753684.350896] LustreError: dumping log to /tmp/lustre-log.1586317108.48439 [1753712.704710] Lustre: fir-OST0036: deleting orphan objects from 0x1080000402:17537637 to 0x1080000402:17537665 [1753729.406848] LustreError: dumping log to /tmp/lustre-log.1586317153.93708 [1753741.694819] LustreError: dumping log to /tmp/lustre-log.1586317165.93662 [1753752.961047] Lustre: fir-OST0030: deleting orphan objects from 0xf00000402:17527717 to 0xf00000402:17527745 [1753755.764559] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1753760.126786] LNet: Service thread pid 50879 was inactive for 601.13s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1753760.143896] LNet: Skipped 4 previous similar messages [1753760.149129] Pid: 50879, comm: ldlm_cn01_030 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753760.159822] Call Trace: [1753760.162466] [] __wait_on_freeing_inode+0xb0/0xf0 [1753760.168944] [] find_inode_fast+0x71/0xb0 [1753760.174725] [] iget_locked+0x79/0x1a0 [1753760.180261] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1753760.186746] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1753760.193244] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1753760.200672] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1753760.207668] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1753760.214864] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1753760.221949] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1753760.229489] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1753760.236586] [] lu_object_find+0x16/0x20 [obdclass] [1753760.243265] [] ofd_object_find+0x35/0x100 [ofd] [1753760.249655] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1753760.256133] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1753760.263256] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1753760.270284] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1753760.277389] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753760.285284] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753760.291794] [] kthread+0xd1/0xe0 [1753760.296882] [] ret_from_fork_nospec_begin+0xe/0x21 [1753760.303528] [] 0xffffffffffffffff [1753760.308723] LustreError: dumping log to /tmp/lustre-log.1586317184.50879 [1753762.174804] Pid: 48828, comm: ll_ost02_009 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753762.185423] Call Trace: [1753762.188099] [] ofd_create_hdl+0xcb3/0x20e0 [ofd] [1753762.194631] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753762.201852] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753762.209804] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753762.216365] [] kthread+0xd1/0xe0 [1753762.221524] [] ret_from_fork_nospec_begin+0xe/0x21 [1753762.228223] [] 0xffffffffffffffff [1753762.233483] LustreError: dumping log to /tmp/lustre-log.1586317186.48828 [1753766.031893] Lustre: fir-OST0032: haven't heard from client 9ed38912-482b-4 (at 10.50.1.57@o2ib2) in 227 seconds. I think it's dead, and I am evicting it. exp ffff9feb7e6d7000, cur 1586317190 expire 1586317040 last 1586316963 [1753766.051948] Lustre: Skipped 5 previous similar messages [1753766.270788] Pid: 93794, comm: ll_ost_io00_099 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753766.281730] Call Trace: [1753766.284402] [] __lock_page+0x74/0x90 [1753766.289940] [] __find_lock_page+0x54/0x70 [1753766.295956] [] find_or_create_page+0x34/0xa0 [1753766.302223] [] osd_bufs_get+0x413/0x870 [osd_ldiskfs] [1753766.309273] [] ofd_preprw_write.isra.30+0x476/0xea0 [ofd] [1753766.316689] [] ofd_preprw+0x41f/0x1240 [ofd] [1753766.322961] [] tgt_brw_write+0xd33/0x1d60 [ptlrpc] [1753766.329792] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753766.337043] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753766.345090] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753766.351696] [] kthread+0xd1/0xe0 [1753766.356895] [] ret_from_fork_nospec_begin+0xe/0x21 [1753766.363685] [] 0xffffffffffffffff [1753766.369044] LustreError: dumping log to /tmp/lustre-log.1586317190.93794 [1753766.376674] Pid: 93568, comm: ll_ost_io00_084 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753766.387585] Call Trace: [1753766.390284] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1753766.397385] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1753766.404725] [] start_this_handle+0x1a1/0x430 [jbd2] [1753766.411505] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1753766.418421] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1753766.426055] [] osd_trans_start+0x20e/0x4e0 [osd_ldiskfs] [1753766.433298] [] ofd_commitrw_write+0xf1e/0x1db0 [ofd] [1753766.440146] [] ofd_commitrw+0x47c/0xa50 [ofd] [1753766.446419] [] tgt_brw_write+0x102d/0x1d60 [ptlrpc] [1753766.453235] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753766.460431] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753766.468367] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753766.474950] [] kthread+0xd1/0xe0 [1753766.480086] [] ret_from_fork_nospec_begin+0xe/0x21 [1753766.486742] [] 0xffffffffffffffff [1753767.144017] Lustre: fir-OST0030: deleting orphan objects from 0x0:91408493 to 0x0:91408609 [1753770.366779] Pid: 93813, comm: ll_ost_io00_103 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1753770.377683] Call Trace: [1753770.380324] [] __lock_page+0x74/0x90 [1753770.385761] [] __find_lock_page+0x54/0x70 [1753770.391655] [] find_or_create_page+0x34/0xa0 [1753770.397808] [] osd_bufs_get+0x413/0x870 [osd_ldiskfs] [1753770.404745] [] ofd_preprw_write.isra.30+0x476/0xea0 [ofd] [1753770.412027] [] ofd_preprw+0x41f/0x1240 [ofd] [1753770.418185] [] tgt_brw_write+0xd33/0x1d60 [ptlrpc] [1753770.424923] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1753770.432065] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1753770.440002] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1753770.446537] [] kthread+0xd1/0xe0 [1753770.451626] [] ret_from_fork_nospec_begin+0xe/0x21 [1753770.458299] [] 0xffffffffffffffff [1753770.463506] LustreError: dumping log to /tmp/lustre-log.1586317194.93813 [1753774.462768] LustreError: dumping log to /tmp/lustre-log.1586317198.93815 [1753778.558766] LustreError: dumping log to /tmp/lustre-log.1586317202.93816 [1753781.214636] Lustre: fir-OST0038: deleting orphan objects from 0x1100000401:17527529 to 0x1100000401:17527681 [1753782.654763] LustreError: dumping log to /tmp/lustre-log.1586317206.49412 [1753786.750753] LustreError: dumping log to /tmp/lustre-log.1586317210.49417 [1753803.134739] LustreError: dumping log to /tmp/lustre-log.1586317227.93714 [1753805.940441] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1753856.116307] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1753880.958622] LNet: Service thread pid 49199 was inactive for 1201.91s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1753880.971746] LNet: Skipped 78 previous similar messages [1753880.977093] LustreError: dumping log to /tmp/lustre-log.1586317304.49199 [1753888.863263] Lustre: fir-OST0036: deleting orphan objects from 0x1080000400:29471368 to 0x1080000400:29471393 [1753889.150597] LustreError: dumping log to /tmp/lustre-log.1586317313.48291 [1753893.246597] LustreError: dumping log to /tmp/lustre-log.1586317317.48995 [1753896.429039] Lustre: fir-OST0038: deleting orphan objects from 0x1100000402:8814833 to 0x1100000402:8814913 [1753898.910933] Lustre: fir-OST003a: deleting orphan objects from 0x1180000400:29556535 to 0x1180000400:29556561 [1753899.985583] LustreError: 49078:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586317023, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0030_UUID lock: ffff9fe6370ef080/0xf3793efb89bdccdf lrc: 3/0,1 mode: --/PW res: [0xf00000401:0x86dd12:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 49078 timeout: 0 lvb_type: 0 [1753900.029299] LustreError: 49078:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 8 previous similar messages [1753906.292281] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1753911.148780] Lustre: fir-OST003a: deleting orphan objects from 0x1180000401:8854953 to 0x1180000401:8854977 [1753926.014542] LustreError: dumping log to /tmp/lustre-log.1586317350.49147 [1753930.110547] LustreError: dumping log to /tmp/lustre-log.1586317354.48359 [1753934.206540] LustreError: dumping log to /tmp/lustre-log.1586317358.48924 [1753954.686510] LustreError: dumping log to /tmp/lustre-log.1586317378.49394 [1753956.468222] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1753966.974497] LustreError: dumping log to /tmp/lustre-log.1586317390.49402 [1754006.644049] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1754052.990337] LustreError: dumping log to /tmp/lustre-log.1586317476.49424 [1754056.819961] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1754093.950285] Pid: 49119, comm: ll_ost02_078 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754093.960903] Call Trace: [1754093.963575] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754093.970676] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754093.977965] [] start_this_handle+0x1a1/0x430 [jbd2] [1754093.984704] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754093.991564] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754093.999180] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754094.006417] [] dqget+0x3fa/0x450 [1754094.011510] [] dquot_get_dqblk+0x14/0x1f0 [1754094.017392] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754094.025092] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754094.031665] [] ofd_quotactl+0x13c/0x380 [ofd] [1754094.037912] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754094.045070] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754094.053005] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754094.059507] [] kthread+0xd1/0xe0 [1754094.064637] [] ret_from_fork_nospec_begin+0xe/0x21 [1754094.071285] [] 0xffffffffffffffff [1754094.076507] LustreError: dumping log to /tmp/lustre-log.1586317518.49119 [1754098.046287] Pid: 49065, comm: ll_ost02_066 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754098.056898] Call Trace: [1754098.059545] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1754098.066686] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1754098.074088] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1754098.080840] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1754098.087331] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754098.094481] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754098.102403] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754098.108978] [] kthread+0xd1/0xe0 [1754098.114084] [] ret_from_fork_nospec_begin+0xe/0x21 [1754098.120776] [] 0xffffffffffffffff [1754098.125994] LustreError: dumping log to /tmp/lustre-log.1586317522.49065 [1754102.142275] Pid: 48976, comm: ll_ost00_021 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754102.152880] Call Trace: [1754102.155545] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754102.162626] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754102.169924] [] start_this_handle+0x1a1/0x430 [jbd2] [1754102.176656] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754102.183504] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754102.191114] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754102.198310] [] dqget+0x3fa/0x450 [1754102.203403] [] dquot_get_dqblk+0x14/0x1f0 [1754102.209285] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754102.216982] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754102.223581] [] ofd_quotactl+0x13c/0x380 [ofd] [1754102.229801] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754102.236947] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754102.244846] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754102.251362] [] kthread+0xd1/0xe0 [1754102.256450] [] ret_from_fork_nospec_begin+0xe/0x21 [1754102.263097] [] 0xffffffffffffffff [1754102.268282] LustreError: dumping log to /tmp/lustre-log.1586317526.48976 [1754106.239258] Pid: 48299, comm: ll_ost03_002 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754106.249863] Call Trace: [1754106.252518] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754106.259603] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754106.266876] [] start_this_handle+0x1a1/0x430 [jbd2] [1754106.273608] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754106.280430] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754106.288040] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754106.295205] [] dqget+0x3fa/0x450 [1754106.300292] [] dquot_get_dqblk+0x14/0x1f0 [1754106.306151] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754106.313850] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754106.320419] [] ofd_quotactl+0x13c/0x380 [ofd] [1754106.326630] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754106.333790] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754106.341740] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754106.348270] [] kthread+0xd1/0xe0 [1754106.353390] [] ret_from_fork_nospec_begin+0xe/0x21 [1754106.360042] [] 0xffffffffffffffff [1754106.365226] LustreError: dumping log to /tmp/lustre-log.1586317530.48299 [1754106.995923] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1754118.526242] Pid: 49210, comm: ll_ost02_097 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754118.536859] Call Trace: [1754118.539510] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1754118.546657] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1754118.554062] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1754118.560833] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1754118.567330] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754118.574472] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754118.582370] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754118.588901] [] kthread+0xd1/0xe0 [1754118.593990] [] ret_from_fork_nospec_begin+0xe/0x21 [1754118.600653] [] 0xffffffffffffffff [1754118.605851] LustreError: dumping log to /tmp/lustre-log.1586317542.49210 [1754122.622239] LustreError: dumping log to /tmp/lustre-log.1586317546.49409 [1754171.774164] LustreError: dumping log to /tmp/lustre-log.1586317595.48869 [1754179.966135] LustreError: dumping log to /tmp/lustre-log.1586317603.49191 [1754184.062147] LustreError: dumping log to /tmp/lustre-log.1586317608.49168 [1754201.030126] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9fcc2a351050 x1663062070346880/t0(0) o3->9b444a4b-8a4b-4@10.50.14.3@o2ib2:720/0 lens 488/0 e 0 to 0 dl 1586317630 ref 2 fl New:/2/ffffffff rc 0/-1 [1754201.057208] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1106 previous similar messages [1754204.542099] LustreError: dumping log to /tmp/lustre-log.1586317628.48879 [1754207.347727] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1754207.356781] Lustre: Skipped 5 previous similar messages [1754212.734095] LustreError: dumping log to /tmp/lustre-log.1586317636.49050 [1754225.022071] LustreError: dumping log to /tmp/lustre-log.1586317649.49045 [1754229.118077] LustreError: dumping log to /tmp/lustre-log.1586317653.48971 [1754237.310048] LustreError: dumping log to /tmp/lustre-log.1586317661.49193 [1754244.732230] Lustre: fir-OST0032: deleting orphan objects from 0xf80000402:8704297 to 0xf80000402:8704321 [1754253.693998] LustreError: dumping log to /tmp/lustre-log.1586317677.48288 [1754255.715762] Lustre: fir-OST0038: Client 1dfefd00-0951-4 (at 10.50.4.59@o2ib2) reconnecting [1754255.724209] Lustre: Skipped 642 previous similar messages [1754255.729807] Lustre: fir-OST0038: Connection restored to 1dfefd00-0951-4 (at 10.50.4.59@o2ib2) [1754255.738508] Lustre: Skipped 662 previous similar messages [1754265.981966] LustreError: dumping log to /tmp/lustre-log.1586317689.48970 [1754311.037854] LustreError: dumping log to /tmp/lustre-log.1586317735.48509 [1754343.805765] LustreError: dumping log to /tmp/lustre-log.1586317767.49140 [1754348.630188] Lustre: fir-OST0032: deleting orphan objects from 0xf80000401:29393030 to 0xf80000401:29393057 [1754351.997759] LustreError: dumping log to /tmp/lustre-log.1586317775.49138 [1754355.787975] Lustre: fir-OST0030: deleting orphan objects from 0xf00000401:8838430 to 0xf00000401:8838561 [1754357.875569] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1754357.884619] Lustre: Skipped 9 previous similar messages [1754360.189722] LustreError: dumping log to /tmp/lustre-log.1586317784.48832 [1754364.285722] LustreError: dumping log to /tmp/lustre-log.1586317788.48848 [1754388.861654] LustreError: dumping log to /tmp/lustre-log.1586317812.49022 [1754413.437600] LNet: Service thread pid 49115 was inactive for 1203.67s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1754413.454806] LNet: Skipped 9 previous similar messages [1754413.460043] Pid: 49115, comm: ll_ost00_044 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754413.470648] Call Trace: [1754413.473309] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754413.480419] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754413.487676] [] start_this_handle+0x1a1/0x430 [jbd2] [1754413.494451] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754413.501274] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754413.508905] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754413.516076] [] dqget+0x3fa/0x450 [1754413.521162] [] dquot_get_dqblk+0x14/0x1f0 [1754413.527022] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754413.534745] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754413.541303] [] ofd_quotactl+0x13c/0x380 [ofd] [1754413.547542] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754413.554678] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754413.562588] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754413.569090] [] kthread+0xd1/0xe0 [1754413.574199] [] ret_from_fork_nospec_begin+0xe/0x21 [1754413.580851] [] 0xffffffffffffffff [1754413.586053] LustreError: dumping log to /tmp/lustre-log.1586317837.49115 [1754433.917538] Pid: 49142, comm: ll_ost00_047 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754433.928143] Call Trace: [1754433.930825] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754433.937904] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754433.945193] [] start_this_handle+0x1a1/0x430 [jbd2] [1754433.951930] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754433.958781] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754433.966384] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754433.973581] [] dqget+0x3fa/0x450 [1754433.978666] [] dquot_get_dqblk+0x14/0x1f0 [1754433.984538] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754433.992238] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754433.998830] [] ofd_quotactl+0x13c/0x380 [ofd] [1754434.005047] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754434.012195] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754434.020102] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754434.026600] [] kthread+0xd1/0xe0 [1754434.031704] [] ret_from_fork_nospec_begin+0xe/0x21 [1754434.038354] [] 0xffffffffffffffff [1754434.043554] LustreError: dumping log to /tmp/lustre-log.1586317858.49142 [1754478.973408] Pid: 49066, comm: ll_ost02_067 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754478.984035] Call Trace: [1754478.986674] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1754478.993824] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1754479.001193] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1754479.007945] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1754479.014418] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754479.021537] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754479.029431] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754479.035931] [] kthread+0xd1/0xe0 [1754479.041017] [] ret_from_fork_nospec_begin+0xe/0x21 [1754479.047665] [] 0xffffffffffffffff [1754479.052859] LustreError: dumping log to /tmp/lustre-log.1586317903.49066 [1754515.837237] Pid: 48296, comm: ll_ost02_002 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754515.847866] Call Trace: [1754515.850513] [] ofd_create_hdl+0xcb3/0x20e0 [ofd] [1754515.857171] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754515.864346] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754515.872311] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754515.878951] [] kthread+0xd1/0xe0 [1754515.884081] [] ret_from_fork_nospec_begin+0xe/0x21 [1754515.890755] [] 0xffffffffffffffff [1754515.895985] LustreError: dumping log to /tmp/lustre-log.1586317939.48296 [1754544.509170] Pid: 49028, comm: ll_ost02_055 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754544.519780] Call Trace: [1754544.522418] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1754544.529545] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1754544.536962] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1754544.543739] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1754544.550257] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754544.557433] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754544.565387] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754544.571939] [] kthread+0xd1/0xe0 [1754544.577068] [] ret_from_fork_nospec_begin+0xe/0x21 [1754544.583760] [] 0xffffffffffffffff [1754544.589007] LustreError: dumping log to /tmp/lustre-log.1586317968.49028 [1754555.581551] Lustre: fir-OST003a: deleting orphan objects from 0x1180000400:29556566 to 0x1180000400:29556593 [1754556.797143] LNet: Service thread pid 49180 was inactive for 1203.07s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1754556.810296] LNet: Skipped 38 previous similar messages [1754556.815616] LustreError: dumping log to /tmp/lustre-log.1586317980.49180 [1754560.893132] LustreError: dumping log to /tmp/lustre-log.1586317984.49151 [1754564.989125] LustreError: dumping log to /tmp/lustre-log.1586317988.49068 [1754565.111117] LustreError: 48845:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586317689, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0030_UUID lock: ffff9fe15da05e80/0xf3793efb89bf580b lrc: 3/0,1 mode: --/PW res: [0xf00000401:0x86dd0e:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48845 timeout: 0 lvb_type: 0 [1754565.154832] LustreError: 48845:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 7 previous similar messages [1754581.373076] LustreError: dumping log to /tmp/lustre-log.1586318005.49044 [1754589.565058] LustreError: dumping log to /tmp/lustre-log.1586318013.49413 [1754593.661054] LustreError: dumping log to /tmp/lustre-log.1586318017.49396 [1754597.757033] LustreError: dumping log to /tmp/lustre-log.1586318021.49201 [1754601.853027] LustreError: dumping log to /tmp/lustre-log.1586318025.49018 [1754616.634869] Lustre: fir-OST0034: Export ffff9fe79de07000 already connecting from 10.50.4.29@o2ib2 [1754616.643922] Lustre: Skipped 13 previous similar messages [1754644.594944] Lustre: fir-OST0036: deleting orphan objects from 0x1080000400:29471398 to 0x1080000400:29471425 [1754652.667132] Lustre: fir-OST0038: deleting orphan objects from 0x1100000402:8814919 to 0x1100000402:8814945 [1754667.195123] Lustre: fir-OST003a: deleting orphan objects from 0x1180000401:8854982 to 0x1180000401:8855009 [1754687.868814] LustreError: dumping log to /tmp/lustre-log.1586318111.112657 [1754696.060791] LustreError: dumping log to /tmp/lustre-log.1586318120.48865 [1754699.877154] Lustre: fir-OST0030: deleting orphan objects from 0xf00000400:29740262 to 0xf00000400:29740833 [1754704.252774] LustreError: dumping log to /tmp/lustre-log.1586318128.50701 [1754708.348764] LustreError: dumping log to /tmp/lustre-log.1586318132.48935 [1754720.636738] Pid: 49070, comm: ll_ost03_064 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754720.647345] Call Trace: [1754720.650003] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754720.657086] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754720.664354] [] start_this_handle+0x1a1/0x430 [jbd2] [1754720.671090] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754720.677927] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754720.685538] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754720.692720] [] dqget+0x3fa/0x450 [1754720.697811] [] dquot_get_dqblk+0x14/0x1f0 [1754720.703668] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754720.711382] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754720.717943] [] ofd_quotactl+0x13c/0x380 [ofd] [1754720.724173] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754720.731298] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754720.739200] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754720.745702] [] kthread+0xd1/0xe0 [1754720.750805] [] ret_from_fork_nospec_begin+0xe/0x21 [1754720.757455] [] 0xffffffffffffffff [1754720.762653] LustreError: dumping log to /tmp/lustre-log.1586318144.49070 [1754728.828704] Pid: 112661, comm: ll_ost01_103 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754728.839399] Call Trace: [1754728.842055] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754728.849138] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754728.856417] [] start_this_handle+0x1a1/0x430 [jbd2] [1754728.863151] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754728.869973] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754728.877583] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754728.884750] [] dqget+0x3fa/0x450 [1754728.889835] [] dquot_get_dqblk+0x14/0x1f0 [1754728.895703] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754728.903399] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754728.909961] [] ofd_quotactl+0x13c/0x380 [ofd] [1754728.916175] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754728.923307] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754728.931192] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754728.937693] [] kthread+0xd1/0xe0 [1754728.942780] [] ret_from_fork_nospec_begin+0xe/0x21 [1754728.949428] [] 0xffffffffffffffff [1754728.954614] LustreError: dumping log to /tmp/lustre-log.1586318152.112661 [1754732.924695] Pid: 49177, comm: ll_ost01_071 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754732.935299] Call Trace: [1754732.937956] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754732.945038] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754732.952310] [] start_this_handle+0x1a1/0x430 [jbd2] [1754732.959043] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754732.965884] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754732.973507] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754732.980692] [] dqget+0x3fa/0x450 [1754732.985781] [] dquot_get_dqblk+0x14/0x1f0 [1754732.991647] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754732.999361] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754733.005923] [] ofd_quotactl+0x13c/0x380 [ofd] [1754733.012151] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754733.019278] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754733.027183] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754733.033682] [] kthread+0xd1/0xe0 [1754733.038785] [] ret_from_fork_nospec_begin+0xe/0x21 [1754733.045434] [] 0xffffffffffffffff [1754733.050634] LustreError: dumping log to /tmp/lustre-log.1586318157.49177 [1754733.058062] Pid: 48923, comm: ll_ost03_036 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754733.068696] Call Trace: [1754733.071336] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754733.078417] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754733.085708] [] start_this_handle+0x1a1/0x430 [jbd2] [1754733.092441] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754733.099291] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754733.106905] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754733.114107] [] dqget+0x3fa/0x450 [1754733.119195] [] dquot_get_dqblk+0x14/0x1f0 [1754733.125080] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754733.132791] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754733.139354] [] ofd_quotactl+0x13c/0x380 [ofd] [1754733.145582] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754733.152714] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754733.160603] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754733.167122] [] kthread+0xd1/0xe0 [1754733.172223] [] ret_from_fork_nospec_begin+0xe/0x21 [1754733.178888] [] 0xffffffffffffffff [1754741.116667] Pid: 50696, comm: ll_ost01_087 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1754741.127274] Call Trace: [1754741.129930] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1754741.137014] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1754741.144268] [] start_this_handle+0x1a1/0x430 [jbd2] [1754741.151006] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1754741.157831] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1754741.165443] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1754741.172643] [] dqget+0x3fa/0x450 [1754741.177732] [] dquot_get_dqblk+0x14/0x1f0 [1754741.183605] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1754741.191303] [] lquotactl_slv+0x27d/0xa00 [lquota] [1754741.197877] [] ofd_quotactl+0x13c/0x380 [ofd] [1754741.204094] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1754741.211227] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1754741.219129] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1754741.225661] [] kthread+0xd1/0xe0 [1754741.230755] [] ret_from_fork_nospec_begin+0xe/0x21 [1754741.237424] [] 0xffffffffffffffff [1754741.242610] LustreError: dumping log to /tmp/lustre-log.1586318165.50696 [1754745.212662] LustreError: dumping log to /tmp/lustre-log.1586318169.49056 [1754765.692607] LustreError: dumping log to /tmp/lustre-log.1586318189.50699 [1754773.884589] LustreError: dumping log to /tmp/lustre-log.1586318197.48293 [1754777.980578] LustreError: dumping log to /tmp/lustre-log.1586318201.49017 [1754798.460526] LustreError: dumping log to /tmp/lustre-log.1586318222.48897 [1754801.280529] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-101), not sending early reply req@ffff9fdbfb6fb050 x1659566450471936/t0(0) o4->9cb1f402-15f9-4@10.50.7.31@o2ib2:565/0 lens 488/0 e 0 to 0 dl 1586318230 ref 2 fl New:/2/ffffffff rc 0/-1 [1754801.307874] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 1524 previous similar messages [1754802.556517] LustreError: dumping log to /tmp/lustre-log.1586318226.49078 [1754806.652505] LustreError: dumping log to /tmp/lustre-log.1586318230.49120 [1754810.748514] LustreError: dumping log to /tmp/lustre-log.1586318234.49223 [1754814.844485] LustreError: dumping log to /tmp/lustre-log.1586318238.112655 [1754818.940473] LustreError: dumping log to /tmp/lustre-log.1586318242.48913 [1754855.781394] Lustre: fir-OST0038: Client b86675a4-f918-4 (at 10.50.8.50@o2ib2) reconnecting [1754855.789839] Lustre: Skipped 699 previous similar messages [1754855.795432] Lustre: fir-OST0038: Connection restored to b86675a4-f918-4 (at 10.50.8.50@o2ib2) [1754855.804127] Lustre: Skipped 704 previous similar messages [1754855.804384] LustreError: dumping log to /tmp/lustre-log.1586318279.49162 [1754881.012721] Lustre: fir-OST0038: deleting orphan objects from 0x1100000400:29343444 to 0x1100000400:29344193 [1754909.052228] LustreError: dumping log to /tmp/lustre-log.1586318333.49144 [1754933.628154] LustreError: dumping log to /tmp/lustre-log.1586318357.49205 [1754937.724144] LustreError: dumping log to /tmp/lustre-log.1586318361.48847 [1754945.917121] LustreError: dumping log to /tmp/lustre-log.1586318369.48952 [1754950.012112] LustreError: dumping log to /tmp/lustre-log.1586318374.49047 [1754954.108106] LustreError: dumping log to /tmp/lustre-log.1586318378.48957 [1754966.396068] LustreError: dumping log to /tmp/lustre-log.1586318390.48989 [1754974.588059] LustreError: dumping log to /tmp/lustre-log.1586318398.48852 [1754978.684037] LustreError: dumping log to /tmp/lustre-log.1586318402.49037 [1755001.082235] Lustre: fir-OST0032: deleting orphan objects from 0xf80000402:8704325 to 0xf80000402:8704353 [1755011.451953] LustreError: dumping log to /tmp/lustre-log.1586318435.112664 [1755015.547952] LustreError: dumping log to /tmp/lustre-log.1586318439.48314 [1755023.739909] LNet: Service thread pid 48945 was inactive for 1204.60s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1755023.757106] LNet: Skipped 9 previous similar messages [1755023.762343] Pid: 48945, comm: ll_ost03_042 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755023.772985] Call Trace: [1755023.775647] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755023.782729] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755023.790018] [] start_this_handle+0x1a1/0x430 [jbd2] [1755023.796754] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755023.803573] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755023.811199] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755023.818367] [] dqget+0x3fa/0x450 [1755023.823491] [] dquot_get_dqblk+0x14/0x1f0 [1755023.829349] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755023.837076] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755023.843647] [] ofd_quotactl+0x13c/0x380 [ofd] [1755023.849892] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755023.857021] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755023.864922] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755023.871425] [] kthread+0xd1/0xe0 [1755023.876525] [] ret_from_fork_nospec_begin+0xe/0x21 [1755023.883176] [] 0xffffffffffffffff [1755023.888390] LustreError: dumping log to /tmp/lustre-log.1586318447.48945 [1755036.027867] Pid: 49141, comm: ll_ost01_065 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755036.038470] Call Trace: [1755036.041147] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755036.048230] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755036.055527] [] start_this_handle+0x1a1/0x430 [jbd2] [1755036.062259] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755036.069114] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755036.076724] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755036.083936] [] dqget+0x3fa/0x450 [1755036.089036] [] dquot_get_dqblk+0x14/0x1f0 [1755036.094939] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755036.102636] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755036.109231] [] ofd_quotactl+0x13c/0x380 [ofd] [1755036.115447] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755036.122599] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755036.130499] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755036.137030] [] kthread+0xd1/0xe0 [1755036.142123] [] ret_from_fork_nospec_begin+0xe/0x21 [1755036.148783] [] 0xffffffffffffffff [1755036.154000] LustreError: dumping log to /tmp/lustre-log.1586318460.49141 [1755040.123874] Pid: 48991, comm: ll_ost02_047 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755040.134485] Call Trace: [1755040.137149] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755040.144233] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755040.151516] [] start_this_handle+0x1a1/0x430 [jbd2] [1755040.158254] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755040.165088] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755040.172702] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755040.179908] [] dqget+0x3fa/0x450 [1755040.184999] [] dquot_get_dqblk+0x14/0x1f0 [1755040.190872] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755040.198604] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755040.205167] [] ofd_quotactl+0x13c/0x380 [ofd] [1755040.211412] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755040.218549] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755040.226475] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755040.232980] [] kthread+0xd1/0xe0 [1755040.238080] [] ret_from_fork_nospec_begin+0xe/0x21 [1755040.244746] [] 0xffffffffffffffff [1755040.249947] LustreError: dumping log to /tmp/lustre-log.1586318464.48991 [1755048.315832] Pid: 48967, comm: ll_ost02_040 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755048.326440] Call Trace: [1755048.329096] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755048.336180] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755048.343450] [] start_this_handle+0x1a1/0x430 [jbd2] [1755048.350184] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755048.357018] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755048.364632] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755048.371815] [] dqget+0x3fa/0x450 [1755048.376903] [] dquot_get_dqblk+0x14/0x1f0 [1755048.382764] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755048.390498] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755048.397061] [] ofd_quotactl+0x13c/0x380 [ofd] [1755048.403291] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755048.410425] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755048.418329] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755048.424829] [] kthread+0xd1/0xe0 [1755048.429932] [] ret_from_fork_nospec_begin+0xe/0x21 [1755048.436595] [] 0xffffffffffffffff [1755048.441800] LustreError: dumping log to /tmp/lustre-log.1586318472.48967 [1755064.699790] Pid: 48873, comm: ll_ost02_022 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755064.710397] Call Trace: [1755064.713063] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755064.720162] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755064.727431] [] start_this_handle+0x1a1/0x430 [jbd2] [1755064.734168] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755064.741015] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755064.748622] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755064.755827] [] dqget+0x3fa/0x450 [1755064.760913] [] dquot_get_dqblk+0x14/0x1f0 [1755064.766793] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755064.774522] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755064.781110] [] ofd_quotactl+0x13c/0x380 [ofd] [1755064.787341] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755064.794479] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755064.802394] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755064.808901] [] kthread+0xd1/0xe0 [1755064.814015] [] ret_from_fork_nospec_begin+0xe/0x21 [1755064.820659] [] 0xffffffffffffffff [1755064.825859] LustreError: dumping log to /tmp/lustre-log.1586318488.48873 [1755072.891770] LustreError: dumping log to /tmp/lustre-log.1586318496.48961 [1755076.987761] LustreError: dumping log to /tmp/lustre-log.1586318500.48915 [1755081.083746] LustreError: dumping log to /tmp/lustre-log.1586318505.49087 [1755104.124118] Lustre: fir-OST0032: deleting orphan objects from 0xf80000401:29393060 to 0xf80000401:29393089 [1755111.799606] Lustre: fir-OST0030: deleting orphan objects from 0xf00000401:8838565 to 0xf00000401:8838593 [1755135.601269] Lustre: fir-OST0034: Export ffffa0090f6d5000 already connecting from 10.50.1.57@o2ib2 [1755135.610318] Lustre: Skipped 32 previous similar messages [1755146.619581] LustreError: dumping log to /tmp/lustre-log.1586318570.48825 [1755163.003593] LNet: Service thread pid 49221 was inactive for 1200.98s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1755163.016714] LNet: Skipped 51 previous similar messages [1755163.022036] LustreError: dumping log to /tmp/lustre-log.1586318587.49221 [1755167.099517] LustreError: dumping log to /tmp/lustre-log.1586318591.48949 [1755183.483488] LustreError: dumping log to /tmp/lustre-log.1586318607.49411 [1755191.675446] LustreError: dumping log to /tmp/lustre-log.1586318615.48993 [1755203.963412] LustreError: dumping log to /tmp/lustre-log.1586318627.48985 [1755208.059418] LustreError: dumping log to /tmp/lustre-log.1586318632.49000 [1755212.155389] LustreError: dumping log to /tmp/lustre-log.1586318636.49195 [1755212.313488] Lustre: fir-OST003a: deleting orphan objects from 0x1180000400:29556595 to 0x1180000400:29556625 [1755220.347361] LustreError: dumping log to /tmp/lustre-log.1586318644.48870 [1755224.443359] LustreError: dumping log to /tmp/lustre-log.1586318648.49136 [1755228.539336] LustreError: dumping log to /tmp/lustre-log.1586318652.49063 [1755236.731320] LustreError: dumping log to /tmp/lustre-log.1586318660.49054 [1755240.828302] LustreError: dumping log to /tmp/lustre-log.1586318664.48684 [1755244.923302] LustreError: dumping log to /tmp/lustre-log.1586318668.48972 [1755246.231299] LustreError: 48996:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586318370, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST003a_UUID lock: ffff9fe1303bd580/0xf3793efb89c1ff66 lrc: 3/0,1 mode: --/PW res: [0x1180000400:0x1c2ff53:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48996 timeout: 0 lvb_type: 0 [1755246.275221] LustreError: 48996:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 6 previous similar messages [1755249.019291] LustreError: dumping log to /tmp/lustre-log.1586318673.49407 [1755253.115274] LustreError: dumping log to /tmp/lustre-log.1586318677.50704 [1755257.211259] LustreError: dumping log to /tmp/lustre-log.1586318681.48868 [1755261.307263] LustreError: dumping log to /tmp/lustre-log.1586318685.49105 [1755265.403231] LustreError: dumping log to /tmp/lustre-log.1586318689.48836 [1755269.499202] LustreError: dumping log to /tmp/lustre-log.1586318693.49165 [1755273.595189] LustreError: dumping log to /tmp/lustre-log.1586318697.49160 [1755277.691161] LustreError: dumping log to /tmp/lustre-log.1586318701.48853 [1755281.787146] LustreError: dumping log to /tmp/lustre-log.1586318705.49001 [1755285.883133] LustreError: dumping log to /tmp/lustre-log.1586318709.49101 [1755289.979123] LustreError: dumping log to /tmp/lustre-log.1586318713.49186 [1755326.843027] Pid: 49416, comm: ll_ost01_084 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755326.853632] Call Trace: [1755326.856271] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1755326.863417] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1755326.870796] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1755326.877555] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1755326.884044] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755326.891194] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755326.899085] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755326.905606] [] kthread+0xd1/0xe0 [1755326.910712] [] ret_from_fork_nospec_begin+0xe/0x21 [1755326.917383] [] 0xffffffffffffffff [1755326.922574] LustreError: dumping log to /tmp/lustre-log.1586318750.49416 [1755355.514976] Pid: 48861, comm: ll_ost01_016 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755355.525578] Call Trace: [1755355.528233] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755355.535319] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755355.542609] [] start_this_handle+0x1a1/0x430 [jbd2] [1755355.549350] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755355.556209] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755355.563822] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755355.571007] [] dqget+0x3fa/0x450 [1755355.576096] [] dquot_get_dqblk+0x14/0x1f0 [1755355.581987] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755355.589684] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755355.596286] [] ofd_quotactl+0x13c/0x380 [ofd] [1755355.602502] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755355.609664] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755355.617557] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755355.624121] [] kthread+0xd1/0xe0 [1755355.629213] [] ret_from_fork_nospec_begin+0xe/0x21 [1755355.635891] [] 0xffffffffffffffff [1755355.641088] LustreError: dumping log to /tmp/lustre-log.1586318779.48861 [1755400.171299] Lustre: fir-OST0036: deleting orphan objects from 0x1080000400:29471428 to 0x1080000400:29471457 [1755400.570841] Pid: 49442, comm: ldlm_cn02_013 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755400.581534] Call Trace: [1755400.584183] [] __wait_on_freeing_inode+0xb0/0xf0 [1755400.590662] [] find_inode_fast+0x71/0xb0 [1755400.596462] [] iget_locked+0x79/0x1a0 [1755400.601989] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1755400.608489] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1755400.614990] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1755400.622443] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1755400.629440] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1755400.636622] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1755400.643703] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1755400.651258] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1755400.658360] [] lu_object_find+0x16/0x20 [obdclass] [1755400.665040] [] ofd_object_find+0x35/0x100 [ofd] [1755400.671443] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1755400.677943] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1755400.685077] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1755400.692114] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1755400.699225] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755400.707152] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755400.713658] [] kthread+0xd1/0xe0 [1755400.718749] [] ret_from_fork_nospec_begin+0xe/0x21 [1755400.725417] [] 0xffffffffffffffff [1755400.730617] LustreError: dumping log to /tmp/lustre-log.1586318824.49442 [1755401.356864] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9fcceb729050 x1659633128102912/t0(0) o3->cae40feb-4786-4@10.50.4.14@o2ib2:410/0 lens 488/0 e 0 to 0 dl 1586318830 ref 2 fl New:/2/ffffffff rc 0/-1 [1755401.383972] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 2218 previous similar messages [1755408.762819] Pid: 72774, comm: ldlm_cn00_027 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755408.773515] Call Trace: [1755408.776182] [] lu_object_find_at+0x847/0xa60 [obdclass] [1755408.783287] [] lu_object_find+0x16/0x20 [obdclass] [1755408.789975] [] ofd_object_find+0x35/0x100 [ofd] [1755408.796366] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1755408.802872] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1755408.809990] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1755408.817044] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1755408.824151] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755408.832084] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755408.838587] [] kthread+0xd1/0xe0 [1755408.843697] [] ret_from_fork_nospec_begin+0xe/0x21 [1755408.850358] [] 0xffffffffffffffff [1755408.855572] LustreError: dumping log to /tmp/lustre-log.1586318832.72774 [1755409.577118] Lustre: fir-OST0038: deleting orphan objects from 0x1100000402:8814950 to 0x1100000402:8814977 [1755410.810814] Pid: 50881, comm: ldlm_cn01_031 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755410.821510] Call Trace: [1755410.824170] [] __wait_on_freeing_inode+0xb0/0xf0 [1755410.830650] [] find_inode_fast+0x71/0xb0 [1755410.836452] [] iget_locked+0x79/0x1a0 [1755410.841973] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1755410.848474] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1755410.854963] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1755410.862420] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1755410.869420] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1755410.876617] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1755410.883703] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1755410.891263] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1755410.898367] [] lu_object_find+0x16/0x20 [obdclass] [1755410.905051] [] ofd_object_find+0x35/0x100 [ofd] [1755410.911444] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1755410.917938] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1755410.925069] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1755410.932117] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1755410.939231] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755410.947164] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755410.953669] [] kthread+0xd1/0xe0 [1755410.958755] [] ret_from_fork_nospec_begin+0xe/0x21 [1755410.965440] [] 0xffffffffffffffff [1755410.970640] LustreError: dumping log to /tmp/lustre-log.1586318834.50881 [1755423.617066] Lustre: fir-OST003a: deleting orphan objects from 0x1180000401:8855014 to 0x1180000401:8855041 [1755425.146761] LustreError: dumping log to /tmp/lustre-log.1586318849.130090 [1755429.242754] LustreError: dumping log to /tmp/lustre-log.1586318853.49025 [1755431.290744] LustreError: dumping log to /tmp/lustre-log.1586318855.49476 [1755433.338740] LustreError: dumping log to /tmp/lustre-log.1586318857.49454 [1755441.530717] LustreError: dumping log to /tmp/lustre-log.1586318865.49157 [1755445.626706] LustreError: dumping log to /tmp/lustre-log.1586318869.50958 [1755455.891001] Lustre: fir-OST0030: Client fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) reconnecting [1755455.900137] Lustre: Skipped 1305 previous similar messages [1755455.905825] Lustre: fir-OST0030: Connection restored to fir-MDT0001-mdtlov_UUID (at 10.0.10.52@o2ib7) [1755455.915218] Lustre: Skipped 1310 previous similar messages [1755455.921071] Lustre: fir-OST0030: deleting orphan objects from 0xf00000400:29740836 to 0xf00000400:29740865 [1755457.914675] LustreError: dumping log to /tmp/lustre-log.1586318881.48846 [1755466.106651] LustreError: dumping log to /tmp/lustre-log.1586318890.48845 [1755470.202637] LustreError: dumping log to /tmp/lustre-log.1586318894.49438 [1755476.346632] LustreError: dumping log to /tmp/lustre-log.1586318900.50855 [1755490.682599] LustreError: dumping log to /tmp/lustre-log.1586318914.48270 [1755498.874558] LustreError: dumping log to /tmp/lustre-log.1586318922.50856 [1755527.546476] LustreError: dumping log to /tmp/lustre-log.1586318951.48265 [1755603.322239] LustreError: dumping log to /tmp/lustre-log.1586319027.49429 [1755605.370233] LustreError: dumping log to /tmp/lustre-log.1586319029.49166 [1755609.466223] LustreError: dumping log to /tmp/lustre-log.1586319033.48867 [1755613.562206] LustreError: dumping log to /tmp/lustre-log.1586319037.49102 [1755617.658200] LustreError: dumping log to /tmp/lustre-log.1586319041.50702 [1755623.802173] LustreError: dumping log to /tmp/lustre-log.1586319047.49453 [1755627.898163] LNet: Service thread pid 50878 was inactive for 709.33s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1755627.915275] LNet: Skipped 9 previous similar messages [1755627.920507] Pid: 50878, comm: ldlm_cn01_029 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755627.931198] Call Trace: [1755627.933844] [] __wait_on_freeing_inode+0xb0/0xf0 [1755627.940356] [] find_inode_fast+0x71/0xb0 [1755627.946165] [] iget_locked+0x79/0x1a0 [1755627.951674] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1755627.958159] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1755627.964641] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1755627.972074] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1755627.979079] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1755627.986247] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1755627.993337] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1755628.000877] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1755628.007975] [] lu_object_find+0x16/0x20 [obdclass] [1755628.014640] [] ofd_object_find+0x35/0x100 [ofd] [1755628.021025] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1755628.027502] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1755628.034614] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1755628.041636] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1755628.048739] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755628.056637] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755628.063136] [] kthread+0xd1/0xe0 [1755628.068224] [] ret_from_fork_nospec_begin+0xe/0x21 [1755628.074870] [] 0xffffffffffffffff [1755628.080065] LustreError: dumping log to /tmp/lustre-log.1586319052.50878 [1755637.418631] Lustre: fir-OST0038: deleting orphan objects from 0x1100000400:29344195 to 0x1100000400:29344225 [1755691.385964] Pid: 48872, comm: ll_ost03_022 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755691.396567] Call Trace: [1755691.399208] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1755691.406325] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1755691.413708] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1755691.420442] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1755691.426933] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755691.434048] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755691.441950] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755691.448452] [] kthread+0xd1/0xe0 [1755691.453556] [] ret_from_fork_nospec_begin+0xe/0x21 [1755691.460205] [] 0xffffffffffffffff [1755691.465429] LustreError: dumping log to /tmp/lustre-log.1586319115.48872 [1755695.481952] Pid: 68487, comm: ldlm_cn00_024 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755695.492675] Call Trace: [1755695.495325] [] __wait_on_freeing_inode+0xb0/0xf0 [1755695.501860] [] find_inode_fast+0x71/0xb0 [1755695.507703] [] iget_locked+0x79/0x1a0 [1755695.513222] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1755695.519767] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1755695.526249] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1755695.533705] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1755695.540718] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1755695.547936] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1755695.555020] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1755695.562611] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1755695.569711] [] lu_object_find+0x16/0x20 [obdclass] [1755695.576438] [] ofd_object_find+0x35/0x100 [ofd] [1755695.582823] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1755695.589339] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1755695.596466] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1755695.603534] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1755695.610647] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755695.618595] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755695.625108] [] kthread+0xd1/0xe0 [1755695.630253] [] ret_from_fork_nospec_begin+0xe/0x21 [1755695.636904] [] 0xffffffffffffffff [1755695.642136] LustreError: dumping log to /tmp/lustre-log.1586319119.68487 [1755752.031024] Lustre: fir-OST0034: Export ffff9fe2ca6d5000 already connecting from 10.50.10.31@o2ib2 [1755752.040159] Lustre: Skipped 52 previous similar messages [1755757.373711] Lustre: fir-OST0032: deleting orphan objects from 0xf80000402:8704361 to 0xf80000402:8704385 [1755769.209732] Pid: 48927, comm: ll_ost00_017 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755769.220335] Call Trace: [1755769.223000] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755769.230099] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755769.237371] [] start_this_handle+0x1a1/0x430 [jbd2] [1755769.244120] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755769.250957] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755769.258592] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755769.265777] [] dqget+0x3fa/0x450 [1755769.270878] [] dquot_get_dqblk+0x14/0x1f0 [1755769.276733] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755769.284438] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755769.291026] [] ofd_quotactl+0x13c/0x380 [ofd] [1755769.297240] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755769.304387] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755769.312286] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755769.318819] [] kthread+0xd1/0xe0 [1755769.323928] [] ret_from_fork_nospec_begin+0xe/0x21 [1755769.330578] [] 0xffffffffffffffff [1755769.335786] LustreError: dumping log to /tmp/lustre-log.1586319193.48927 [1755769.344033] Pid: 48921, comm: ll_ost03_035 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755769.354688] Call Trace: [1755769.357326] [] ofd_create_hdl+0xcb3/0x20e0 [ofd] [1755769.363802] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755769.370941] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755769.378862] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755769.385380] [] kthread+0xd1/0xe0 [1755769.390467] [] ret_from_fork_nospec_begin+0xe/0x21 [1755769.397122] [] 0xffffffffffffffff [1755783.545682] LNet: Service thread pid 49465 was inactive for 859.51s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1755783.558721] LNet: Skipped 59 previous similar messages [1755783.564044] LustreError: dumping log to /tmp/lustre-log.1586319207.49465 [1755785.593680] LustreError: dumping log to /tmp/lustre-log.1586319209.49441 [1755789.689669] LustreError: dumping log to /tmp/lustre-log.1586319213.49443 [1755797.881642] LustreError: dumping log to /tmp/lustre-log.1586319221.49459 [1755801.977630] LustreError: dumping log to /tmp/lustre-log.1586319225.50873 [1755830.649555] LustreError: dumping log to /tmp/lustre-log.1586319254.48382 [1755847.033490] LustreError: dumping log to /tmp/lustre-log.1586319271.48929 [1755860.111548] Lustre: fir-OST0032: deleting orphan objects from 0xf80000401:29393093 to 0xf80000401:29393121 [1755867.199740] Lustre: fir-OST0030: deleting orphan objects from 0xf00000401:8838601 to 0xf00000401:8838625 [1755869.913817] Lustre: fir-OST003a: deleting orphan objects from 0x1180000400:29556628 to 0x1180000400:29556657 [1755875.705411] LustreError: dumping log to /tmp/lustre-log.1586319299.50730 [1755887.993374] LustreError: dumping log to /tmp/lustre-log.1586319311.50957 [1755920.761273] LustreError: dumping log to /tmp/lustre-log.1586319344.36522 [1755924.857257] LustreError: dumping log to /tmp/lustre-log.1586319348.36525 [1755933.049239] Pid: 49090, comm: ll_ost03_069 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755933.059843] Call Trace: [1755933.062499] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755933.069583] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755933.076853] [] start_this_handle+0x1a1/0x430 [jbd2] [1755933.083588] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755933.090425] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755933.098060] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755933.105242] [] dqget+0x3fa/0x450 [1755933.110332] [] dquot_get_dqblk+0x14/0x1f0 [1755933.116192] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755933.123912] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755933.130498] [] ofd_quotactl+0x13c/0x380 [ofd] [1755933.136712] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755933.143873] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755933.151774] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755933.158293] [] kthread+0xd1/0xe0 [1755933.163398] [] ret_from_fork_nospec_begin+0xe/0x21 [1755933.170062] [] 0xffffffffffffffff [1755933.175268] LustreError: dumping log to /tmp/lustre-log.1586319357.49090 [1755945.337217] Pid: 49099, comm: ll_ost00_041 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755945.347822] Call Trace: [1755945.350493] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1755945.357579] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1755945.364876] [] start_this_handle+0x1a1/0x430 [jbd2] [1755945.371609] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1755945.378455] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1755945.386066] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1755945.393269] [] dqget+0x3fa/0x450 [1755945.398355] [] dquot_get_dqblk+0x14/0x1f0 [1755945.404239] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1755945.411939] [] lquotactl_slv+0x27d/0xa00 [lquota] [1755945.418505] [] ofd_quotactl+0x13c/0x380 [ofd] [1755945.424740] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755945.431878] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755945.439810] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755945.446330] [] kthread+0xd1/0xe0 [1755945.451440] [] ret_from_fork_nospec_begin+0xe/0x21 [1755945.458097] [] 0xffffffffffffffff [1755945.463308] LustreError: dumping log to /tmp/lustre-log.1586319369.49099 [1755953.529178] Pid: 49468, comm: ldlm_cn02_030 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755953.539874] Call Trace: [1755953.542558] [] __wait_on_freeing_inode+0xb0/0xf0 [1755953.549040] [] find_inode_fast+0x71/0xb0 [1755953.554849] [] iget_locked+0x79/0x1a0 [1755953.560368] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1755953.566905] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1755953.573395] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1755953.580846] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1755953.587846] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1755953.595049] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1755953.602134] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1755953.609719] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1755953.616841] [] lu_object_find+0x16/0x20 [obdclass] [1755953.623553] [] ofd_object_find+0x35/0x100 [ofd] [1755953.629967] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1755953.636479] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1755953.643602] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1755953.650664] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1755953.657785] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755953.665732] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755953.672256] [] kthread+0xd1/0xe0 [1755953.677390] [] ret_from_fork_nospec_begin+0xe/0x21 [1755953.684042] [] 0xffffffffffffffff [1755953.689282] LustreError: dumping log to /tmp/lustre-log.1586319377.49468 [1755957.625171] Pid: 48987, comm: ll_ost03_050 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755957.635788] Call Trace: [1755957.638435] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1755957.645557] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1755957.652939] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1755957.659691] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1755957.666165] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1755957.673302] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755957.681201] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755957.687716] [] kthread+0xd1/0xe0 [1755957.692807] [] ret_from_fork_nospec_begin+0xe/0x21 [1755957.699475] [] 0xffffffffffffffff [1755957.704673] LustreError: dumping log to /tmp/lustre-log.1586319381.48987 [1755965.817143] Pid: 49447, comm: ldlm_cn02_017 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1755965.827898] Call Trace: [1755965.830578] [] __wait_on_freeing_inode+0xb0/0xf0 [1755965.837163] [] find_inode_fast+0x71/0xb0 [1755965.843160] [] iget_locked+0x79/0x1a0 [1755965.848866] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1755965.855581] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1755965.862294] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1755965.869960] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1755965.877213] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1755965.884614] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1755965.891942] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1755965.899639] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1755965.906780] [] lu_object_find+0x16/0x20 [obdclass] [1755965.913591] [] ofd_object_find+0x35/0x100 [ofd] [1755965.920221] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1755965.926927] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1755965.934083] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1755965.941146] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1755965.948311] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1755965.956424] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1755965.963071] [] kthread+0xd1/0xe0 [1755965.968323] [] ret_from_fork_nospec_begin+0xe/0x21 [1755965.975185] [] 0xffffffffffffffff [1755965.980588] LustreError: dumping log to /tmp/lustre-log.1586319389.49447 [1755974.009120] LustreError: dumping log to /tmp/lustre-log.1586319398.49439 [1756001.645052] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9fcd52e1a850 x1659567164876928/t0(0) o4->eac7d7e6-7a39-4@10.50.4.71@o2ib2:255/0 lens 488/0 e 0 to 0 dl 1586319430 ref 2 fl New:/2/ffffffff rc 0/-1 [1756001.672181] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 2399 previous similar messages [1756027.256953] LustreError: dumping log to /tmp/lustre-log.1586319451.112653 [1756032.627953] LustreError: 48881:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586319156, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST003a_UUID lock: ffff9fe8fdc9ee40/0xf3793efb89c76840 lrc: 3/0,1 mode: --/PW res: [0x1180000400:0x1c2ff93:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48881 timeout: 0 lvb_type: 0 [1756032.671871] LustreError: 48881:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 8 previous similar messages [1756055.928871] LustreError: dumping log to /tmp/lustre-log.1586319479.49182 [1756056.539157] Lustre: fir-OST0034: Client c8932097-dc54-4 (at 10.50.10.41@o2ib2) reconnecting [1756056.547690] Lustre: Skipped 1397 previous similar messages [1756056.553379] Lustre: fir-OST0034: Connection restored to c8932097-dc54-4 (at 10.50.10.41@o2ib2) [1756056.562203] Lustre: Skipped 1402 previous similar messages [1756076.408811] LustreError: dumping log to /tmp/lustre-log.1586319500.36518 [1756080.504803] LustreError: dumping log to /tmp/lustre-log.1586319504.48426 [1756088.696779] LustreError: dumping log to /tmp/lustre-log.1586319512.48930 [1756092.792783] LustreError: dumping log to /tmp/lustre-log.1586319516.48874 [1756096.888755] LustreError: dumping log to /tmp/lustre-log.1586319520.48888 [1756133.752665] LustreError: dumping log to /tmp/lustre-log.1586319557.49466 [1756150.136617] LustreError: dumping log to /tmp/lustre-log.1586319574.48996 [1756156.465062] Lustre: fir-OST0036: deleting orphan objects from 0x1080000400:29471460 to 0x1080000400:29471489 [1756162.424582] LustreError: dumping log to /tmp/lustre-log.1586319586.48962 [1756165.782840] Lustre: fir-OST0038: deleting orphan objects from 0x1100000402:8814984 to 0x1100000402:8815009 [1756178.808538] LustreError: dumping log to /tmp/lustre-log.1586319602.49089 [1756179.494823] Lustre: fir-OST003a: deleting orphan objects from 0x1180000401:8855048 to 0x1180000401:8855073 [1756199.288487] LustreError: dumping log to /tmp/lustre-log.1586319623.49059 [1756211.576451] LustreError: dumping log to /tmp/lustre-log.1586319635.95007 [1756211.872922] Lustre: fir-OST0030: deleting orphan objects from 0xf00000400:29740869 to 0xf00000400:29740897 [1756236.152405] LNet: Service thread pid 130088 was inactive for 1238.24s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1756236.169723] LNet: Skipped 9 previous similar messages [1756236.174954] Pid: 130088, comm: ldlm_cn00_028 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756236.185793] Call Trace: [1756236.188437] [] lu_object_find_at+0x847/0xa60 [obdclass] [1756236.195557] [] lu_object_find+0x16/0x20 [obdclass] [1756236.202239] [] ofd_object_find+0x35/0x100 [ofd] [1756236.208636] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1756236.215110] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1756236.222224] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1756236.229246] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1756236.236353] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756236.244262] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756236.250809] [] kthread+0xd1/0xe0 [1756236.255933] [] ret_from_fork_nospec_begin+0xe/0x21 [1756236.262587] [] 0xffffffffffffffff [1756236.267815] LustreError: dumping log to /tmp/lustre-log.1586319660.130088 [1756248.440397] Pid: 49422, comm: ldlm_cn00_005 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756248.451095] Call Trace: [1756248.453741] [] __wait_on_freeing_inode+0xb0/0xf0 [1756248.460221] [] find_inode_fast+0x71/0xb0 [1756248.466004] [] iget_locked+0x79/0x1a0 [1756248.471517] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1756248.477991] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1756248.484465] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1756248.491911] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1756248.498908] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1756248.506095] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1756248.513210] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1756248.520753] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1756248.527866] [] lu_object_find+0x16/0x20 [obdclass] [1756248.534574] [] ofd_object_find+0x35/0x100 [ofd] [1756248.540963] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1756248.547483] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1756248.554601] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1756248.561673] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1756248.568785] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756248.576711] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756248.583214] [] kthread+0xd1/0xe0 [1756248.588301] [] ret_from_fork_nospec_begin+0xe/0x21 [1756248.594987] [] 0xffffffffffffffff [1756248.600185] LustreError: dumping log to /tmp/lustre-log.1586319672.49422 [1756252.536385] Pid: 49492, comm: ldlm_cn00_008 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756252.547080] Call Trace: [1756252.549756] [] __wait_on_freeing_inode+0xb0/0xf0 [1756252.556231] [] find_inode_fast+0x71/0xb0 [1756252.562059] [] iget_locked+0x79/0x1a0 [1756252.567576] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1756252.574096] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1756252.580574] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1756252.588019] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1756252.595060] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1756252.602259] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1756252.609403] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1756252.616940] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1756252.624083] [] lu_object_find+0x16/0x20 [obdclass] [1756252.630738] [] ofd_object_find+0x35/0x100 [ofd] [1756252.637125] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1756252.643599] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1756252.650713] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1756252.657750] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1756252.664862] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756252.672754] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756252.679253] [] kthread+0xd1/0xe0 [1756252.684342] [] ret_from_fork_nospec_begin+0xe/0x21 [1756252.691005] [] 0xffffffffffffffff [1756252.696199] LustreError: dumping log to /tmp/lustre-log.1586319676.49492 [1756264.824349] Pid: 49477, comm: ldlm_cn00_007 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756264.835043] Call Trace: [1756264.837714] [] __wait_on_freeing_inode+0xb0/0xf0 [1756264.844187] [] find_inode_fast+0x71/0xb0 [1756264.849956] [] iget_locked+0x79/0x1a0 [1756264.855469] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1756264.861969] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1756264.868449] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1756264.875908] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1756264.882917] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1756264.890134] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1756264.897217] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1756264.904797] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1756264.911915] [] lu_object_find+0x16/0x20 [obdclass] [1756264.918580] [] ofd_object_find+0x35/0x100 [ofd] [1756264.924967] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1756264.931443] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1756264.938577] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1756264.945602] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1756264.952711] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756264.960610] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756264.967112] [] kthread+0xd1/0xe0 [1756264.972204] [] ret_from_fork_nospec_begin+0xe/0x21 [1756264.978889] [] 0xffffffffffffffff [1756264.984086] LustreError: dumping log to /tmp/lustre-log.1586319688.49477 [1756268.920347] Pid: 50955, comm: ldlm_cn00_017 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756268.931041] Call Trace: [1756268.933689] [] __wait_on_freeing_inode+0xb0/0xf0 [1756268.940191] [] find_inode_fast+0x71/0xb0 [1756268.946016] [] iget_locked+0x79/0x1a0 [1756268.951535] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1756268.958062] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1756268.964534] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1756268.972008] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1756268.979008] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1756268.986206] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1756268.993296] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1756269.000879] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1756269.007989] [] lu_object_find+0x16/0x20 [obdclass] [1756269.014697] [] ofd_object_find+0x35/0x100 [ofd] [1756269.021086] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1756269.027586] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1756269.034727] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1756269.041746] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1756269.048854] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756269.056740] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756269.063254] [] kthread+0xd1/0xe0 [1756269.068343] [] ret_from_fork_nospec_begin+0xe/0x21 [1756269.074992] [] 0xffffffffffffffff [1756269.080194] LustreError: dumping log to /tmp/lustre-log.1586319693.50955 [1756285.304297] LustreError: dumping log to /tmp/lustre-log.1586319709.49033 [1756301.688255] LustreError: dumping log to /tmp/lustre-log.1586319725.67394 [1756313.976234] LustreError: dumping log to /tmp/lustre-log.1586319737.49455 [1756326.264199] LustreError: dumping log to /tmp/lustre-log.1586319750.49188 [1756330.360191] LustreError: dumping log to /tmp/lustre-log.1586319754.50875 [1756338.552173] LustreError: dumping log to /tmp/lustre-log.1586319762.49154 [1756342.648162] LustreError: dumping log to /tmp/lustre-log.1586319766.49096 [1756350.840142] LustreError: dumping log to /tmp/lustre-log.1586319774.48838 [1756353.889923] Lustre: fir-OST0034: Export ffffa00072b1c000 already connecting from 10.50.5.51@o2ib2 [1756353.898974] Lustre: Skipped 72 previous similar messages [1756354.936123] LustreError: dumping log to /tmp/lustre-log.1586319778.48840 [1756367.224097] LustreError: dumping log to /tmp/lustre-log.1586319791.95006 [1756393.448486] Lustre: fir-OST0038: deleting orphan objects from 0x1100000400:29344230 to 0x1100000400:29344257 [1756424.567962] LNet: Service thread pid 50956 was inactive for 1389.91s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1756424.581082] LNet: Skipped 48 previous similar messages [1756424.586436] LustreError: dumping log to /tmp/lustre-log.1586319848.50956 [1756445.047910] LustreError: dumping log to /tmp/lustre-log.1586319869.50949 [1756469.623849] LustreError: dumping log to /tmp/lustre-log.1586319893.49185 [1756473.719844] LustreError: dumping log to /tmp/lustre-log.1586319897.49463 [1756481.911826] LustreError: dumping log to /tmp/lustre-log.1586319905.48273 [1756486.007815] LustreError: dumping log to /tmp/lustre-log.1586319910.50868 [1756494.199782] LustreError: dumping log to /tmp/lustre-log.1586319918.50852 [1756498.295773] LustreError: dumping log to /tmp/lustre-log.1586319922.48926 [1756513.595678] Lustre: fir-OST0032: deleting orphan objects from 0xf80000402:8704389 to 0xf80000402:8704417 [1756514.679734] LustreError: dumping log to /tmp/lustre-log.1586319938.49038 [1756522.871717] LustreError: dumping log to /tmp/lustre-log.1586319946.49077 [1756526.413765] Lustre: fir-OST003a: deleting orphan objects from 0x1180000400:29556661 to 0x1180000400:29556689 [1756526.967709] LustreError: dumping log to /tmp/lustre-log.1586319950.36523 [1756555.639634] Pid: 50695, comm: ll_ost01_086 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756555.650238] Call Trace: [1756555.652897] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756555.659979] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756555.667250] [] start_this_handle+0x1a1/0x430 [jbd2] [1756555.673982] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756555.680805] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756555.688411] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756555.695589] [] dqget+0x3fa/0x450 [1756555.700676] [] dquot_get_dqblk+0x14/0x1f0 [1756555.706534] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756555.714231] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756555.720791] [] ofd_quotactl+0x13c/0x380 [ofd] [1756555.727006] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756555.734164] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756555.742059] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756555.748568] [] kthread+0xd1/0xe0 [1756555.753656] [] ret_from_fork_nospec_begin+0xe/0x21 [1756555.760302] [] 0xffffffffffffffff [1756555.765488] LustreError: dumping log to /tmp/lustre-log.1586319979.50695 [1756567.927608] Pid: 48875, comm: ll_ost01_018 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756567.938213] Call Trace: [1756567.940878] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756567.947962] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756567.955215] [] start_this_handle+0x1a1/0x430 [jbd2] [1756567.961950] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756567.968769] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756567.976380] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756567.983547] [] dqget+0x3fa/0x450 [1756567.988632] [] dquot_get_dqblk+0x14/0x1f0 [1756567.994502] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756568.002214] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756568.008776] [] ofd_quotactl+0x13c/0x380 [ofd] [1756568.014990] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756568.022149] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756568.030035] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756568.036533] [] kthread+0xd1/0xe0 [1756568.041621] [] ret_from_fork_nospec_begin+0xe/0x21 [1756568.048260] [] 0xffffffffffffffff [1756568.053445] LustreError: dumping log to /tmp/lustre-log.1586319992.48875 [1756588.407558] Pid: 95001, comm: ll_ost00_077 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756588.418184] Call Trace: [1756588.420846] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756588.427954] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756588.435239] [] start_this_handle+0x1a1/0x430 [jbd2] [1756588.441991] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756588.448829] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756588.456461] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756588.463656] [] dqget+0x3fa/0x450 [1756588.468746] [] dquot_get_dqblk+0x14/0x1f0 [1756588.474606] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756588.482302] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756588.488878] [] ofd_quotactl+0x13c/0x380 [ofd] [1756588.495095] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756588.502279] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756588.510174] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756588.516689] [] kthread+0xd1/0xe0 [1756588.521780] [] ret_from_fork_nospec_begin+0xe/0x21 [1756588.528456] [] 0xffffffffffffffff [1756588.533669] LustreError: dumping log to /tmp/lustre-log.1586320012.95001 [1756592.503555] Pid: 49194, comm: ll_ost01_073 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756592.514156] Call Trace: [1756592.516813] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756592.523898] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756592.531153] [] start_this_handle+0x1a1/0x430 [jbd2] [1756592.537887] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756592.544731] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756592.552341] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756592.559510] [] dqget+0x3fa/0x450 [1756592.564597] [] dquot_get_dqblk+0x14/0x1f0 [1756592.570464] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756592.578188] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756592.584745] [] ofd_quotactl+0x13c/0x380 [ofd] [1756592.590960] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756592.598093] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756592.605987] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756592.612488] [] kthread+0xd1/0xe0 [1756592.617576] [] ret_from_fork_nospec_begin+0xe/0x21 [1756592.624223] [] 0xffffffffffffffff [1756592.629423] LustreError: dumping log to /tmp/lustre-log.1586320016.49194 [1756602.027543] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9fce63491850 x1659567164876928/t0(0) o4->eac7d7e6-7a39-4@10.50.4.71@o2ib2:101/0 lens 488/0 e 0 to 0 dl 1586320031 ref 2 fl New:/2/ffffffff rc 0/-1 [1756602.054619] Lustre: 93840:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 2623 previous similar messages [1756612.983508] Pid: 49156, comm: ll_ost03_085 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756612.994147] Call Trace: [1756612.996811] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756613.003915] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756613.011186] [] start_this_handle+0x1a1/0x430 [jbd2] [1756613.017948] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756613.024768] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756613.032402] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756613.039573] [] dqget+0x3fa/0x450 [1756613.044692] [] dquot_get_dqblk+0x14/0x1f0 [1756613.050576] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756613.058314] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756613.064895] [] ofd_quotactl+0x13c/0x380 [ofd] [1756613.071141] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756613.078278] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756613.086191] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756613.092721] [] kthread+0xd1/0xe0 [1756613.097842] [] ret_from_fork_nospec_begin+0xe/0x21 [1756613.104494] [] 0xffffffffffffffff [1756613.109698] LustreError: dumping log to /tmp/lustre-log.1586320037.49156 [1756616.103915] Lustre: fir-OST0032: deleting orphan objects from 0xf80000401:29393125 to 0xf80000401:29393153 [1756623.181759] Lustre: fir-OST0030: deleting orphan objects from 0xf00000401:8838630 to 0xf00000401:8838657 [1756653.943424] LustreError: dumping log to /tmp/lustre-log.1586320077.49433 [1756657.713138] Lustre: fir-OST0036: Client 03ac069b-e918-4 (at 10.49.19.6@o2ib1) reconnecting [1756657.721590] Lustre: Skipped 1479 previous similar messages [1756657.727290] Lustre: fir-OST0036: Connection restored to 6eb34c30-3804-4 (at 10.49.19.6@o2ib1) [1756657.736033] Lustre: Skipped 1489 previous similar messages [1756662.135411] LustreError: dumping log to /tmp/lustre-log.1586320086.49464 [1756666.232391] LustreError: dumping log to /tmp/lustre-log.1586320090.95204 [1756670.327389] LustreError: dumping log to /tmp/lustre-log.1586320094.50872 [1756678.519361] LustreError: dumping log to /tmp/lustre-log.1586320102.50703 [1756686.711341] LustreError: dumping log to /tmp/lustre-log.1586320110.49036 [1756703.095319] LustreError: dumping log to /tmp/lustre-log.1586320127.95002 [1756739.959233] LustreError: dumping log to /tmp/lustre-log.1586320163.48982 [1756785.015153] LustreError: dumping log to /tmp/lustre-log.1586320209.50951 [1756808.852097] LustreError: 48844:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1586319932, 300s ago); not entering recovery in server code, just going back to sleep ns: filter-fir-OST0030_UUID lock: ffff9fe5e40bf740/0xf3793efb89ca34d2 lrc: 3/0,1 mode: --/PW res: [0xf00000401:0x86ddc2:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 48844 timeout: 0 lvb_type: 0 [1756808.895843] LustreError: 48844:0:(ldlm_request.c:129:ldlm_expired_completion_wait()) Skipped 7 previous similar messages [1756813.687095] LustreError: dumping log to /tmp/lustre-log.1586320237.95288 [1756834.167038] LustreError: dumping log to /tmp/lustre-log.1586320258.50866 [1756842.359012] LustreError: dumping log to /tmp/lustre-log.1586320266.112663 [1756846.455010] LustreError: dumping log to /tmp/lustre-log.1586320270.49158 [1756850.551002] LustreError: dumping log to /tmp/lustre-log.1586320274.49426 [1756854.646978] LustreError: dumping log to /tmp/lustre-log.1586320278.50694 [1756858.742972] LNet: Service thread pid 48912 was inactive for 1200.15s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1756858.760188] LNet: Skipped 9 previous similar messages [1756858.765421] Pid: 48912, comm: ll_ost03_031 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756858.776056] Call Trace: [1756858.778713] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756858.785800] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756858.793086] [] start_this_handle+0x1a1/0x430 [jbd2] [1756858.799839] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756858.806676] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756858.814304] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756858.821485] [] dqget+0x3fa/0x450 [1756858.826628] [] dquot_get_dqblk+0x14/0x1f0 [1756858.832495] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756858.840230] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756858.846795] [] ofd_quotactl+0x13c/0x380 [ofd] [1756858.853044] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756858.860160] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756858.868060] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756858.874566] [] kthread+0xd1/0xe0 [1756858.879681] [] ret_from_fork_nospec_begin+0xe/0x21 [1756858.886354] [] 0xffffffffffffffff [1756858.891558] LustreError: dumping log to /tmp/lustre-log.1586320282.48912 [1756921.141108] Lustre: fir-OST0038: deleting orphan objects from 0x1100000402:8815013 to 0x1100000402:8815041 [1756935.659259] Lustre: fir-OST003a: deleting orphan objects from 0x1180000401:8855077 to 0x1180000401:8855105 [1756936.566798] Pid: 48881, comm: ll_ost01_021 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756936.577407] Call Trace: [1756936.580049] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1756936.587174] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1756936.594558] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1756936.601293] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1756936.607783] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756936.614906] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756936.622794] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756936.629308] [] kthread+0xd1/0xe0 [1756936.634412] [] ret_from_fork_nospec_begin+0xe/0x21 [1756936.641077] [] 0xffffffffffffffff [1756936.646274] LustreError: dumping log to /tmp/lustre-log.1586320360.48881 [1756936.654351] Pid: 50698, comm: ll_ost01_089 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756936.664960] Call Trace: [1756936.667622] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756936.674706] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756936.681983] [] start_this_handle+0x1a1/0x430 [jbd2] [1756936.688719] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756936.695575] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756936.703186] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756936.710378] [] dqget+0x3fa/0x450 [1756936.715466] [] dquot_get_dqblk+0x14/0x1f0 [1756936.721324] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756936.729028] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756936.735617] [] ofd_quotactl+0x13c/0x380 [ofd] [1756936.741845] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756936.748962] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756936.756849] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756936.763349] [] kthread+0xd1/0xe0 [1756936.768428] [] ret_from_fork_nospec_begin+0xe/0x21 [1756936.775074] [] 0xffffffffffffffff [1756944.758779] Pid: 48839, comm: ll_ost03_011 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756944.769384] Call Trace: [1756944.772023] [] ldlm_completion_ast+0x4e5/0x860 [ptlrpc] [1756944.779166] [] ldlm_cli_enqueue_local+0x231/0x830 [ptlrpc] [1756944.786534] [] ofd_destroy_by_fid+0x1d1/0x510 [ofd] [1756944.793288] [] ofd_destroy_hdl+0x257/0x9d0 [ofd] [1756944.799776] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756944.806892] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756944.814795] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756944.821297] [] kthread+0xd1/0xe0 [1756944.826400] [] ret_from_fork_nospec_begin+0xe/0x21 [1756944.833049] [] 0xffffffffffffffff [1756944.838258] LustreError: dumping log to /tmp/lustre-log.1586320368.48839 [1756952.950756] Pid: 95004, comm: ll_ost00_080 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1756952.961361] Call Trace: [1756952.964016] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1756952.971101] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1756952.978355] [] start_this_handle+0x1a1/0x430 [jbd2] [1756952.985092] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1756952.991919] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1756952.999527] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1756953.006713] [] dqget+0x3fa/0x450 [1756953.011801] [] dquot_get_dqblk+0x14/0x1f0 [1756953.017693] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1756953.025390] [] lquotactl_slv+0x27d/0xa00 [lquota] [1756953.031984] [] ofd_quotactl+0x13c/0x380 [ofd] [1756953.038198] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1756953.045322] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1756953.053209] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1756953.059710] [] kthread+0xd1/0xe0 [1756953.064796] [] ret_from_fork_nospec_begin+0xe/0x21 [1756953.071472] [] 0xffffffffffffffff [1756953.076657] LustreError: dumping log to /tmp/lustre-log.1586320377.95004 [1756957.046750] LustreError: dumping log to /tmp/lustre-log.1586320381.95266 [1756969.334719] LustreError: dumping log to /tmp/lustre-log.1586320393.95403 [1756985.718684] LustreError: dumping log to /tmp/lustre-log.1586320409.49207 [1757006.198647] LustreError: dumping log to /tmp/lustre-log.1586320430.49425 [1757014.390619] LustreError: dumping log to /tmp/lustre-log.1586320438.50870 [1757018.486613] LustreError: dumping log to /tmp/lustre-log.1586320442.48994 [1757022.582607] LustreError: dumping log to /tmp/lustre-log.1586320446.112654 [1757026.678593] LNet: Service thread pid 49434 was inactive for 1911.87s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1757026.691711] LNet: Skipped 49 previous similar messages [1757026.697035] LustreError: dumping log to /tmp/lustre-log.1586320450.49434 [1757030.774583] LustreError: dumping log to /tmp/lustre-log.1586320454.50876 [1757034.870572] LustreError: dumping log to /tmp/lustre-log.1586320458.48909 [1757051.254533] LustreError: dumping log to /tmp/lustre-log.1586320475.48986 [1757067.638490] LustreError: dumping log to /tmp/lustre-log.1586320491.49181 [1757071.734484] LustreError: dumping log to /tmp/lustre-log.1586320495.49108 [1757079.926474] LustreError: dumping log to /tmp/lustre-log.1586320503.50697 [1757080.416058] Lustre: fir-OST0034: Export ffffa00072b1c000 already connecting from 10.50.5.51@o2ib2 [1757080.425107] Lustre: Skipped 40 previous similar messages [1757096.310430] LustreError: dumping log to /tmp/lustre-log.1586320520.49064 [1757124.982364] LustreError: dumping log to /tmp/lustre-log.1586320548.95410 [1757129.078359] LustreError: dumping log to /tmp/lustre-log.1586320553.49420 [1757186.422233] Pid: 49440, comm: ldlm_cn02_012 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757186.432924] Call Trace: [1757186.435576] [] __wait_on_freeing_inode+0xb0/0xf0 [1757186.442057] [] find_inode_fast+0x71/0xb0 [1757186.447828] [] iget_locked+0x79/0x1a0 [1757186.453349] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1757186.459831] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1757186.466315] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1757186.473751] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1757186.480753] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1757186.487962] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1757186.495045] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1757186.502600] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1757186.509699] [] lu_object_find+0x16/0x20 [obdclass] [1757186.516392] [] ofd_object_find+0x35/0x100 [ofd] [1757186.522776] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1757186.529281] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1757186.536409] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1757186.543453] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1757186.550566] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757186.558467] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757186.564966] [] kthread+0xd1/0xe0 [1757186.570053] [] ret_from_fork_nospec_begin+0xe/0x21 [1757186.576700] [] 0xffffffffffffffff [1757186.581894] LustreError: dumping log to /tmp/lustre-log.1586320610.49440 [1757186.589784] Pid: 49393, comm: ldlm_cn02_003 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757186.600506] Call Trace: [1757186.603143] [] __wait_on_freeing_inode+0xb0/0xf0 [1757186.609616] [] find_inode_fast+0x71/0xb0 [1757186.615405] [] iget_locked+0x79/0x1a0 [1757186.620919] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1757186.627406] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1757186.633897] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1757186.641341] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1757186.648340] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1757186.655521] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1757186.662604] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1757186.670150] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1757186.677241] [] lu_object_find+0x16/0x20 [obdclass] [1757186.683914] [] ofd_object_find+0x35/0x100 [ofd] [1757186.690302] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1757186.696793] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1757186.703916] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1757186.710944] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1757186.718053] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757186.725955] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757186.732457] [] kthread+0xd1/0xe0 [1757186.737551] [] ret_from_fork_nospec_begin+0xe/0x21 [1757186.744202] [] 0xffffffffffffffff [1757194.614218] Pid: 49112, comm: ll_ost03_075 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757194.624829] Call Trace: [1757194.627494] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757194.634593] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757194.641897] [] start_this_handle+0x1a1/0x430 [jbd2] [1757194.648651] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757194.655480] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757194.663099] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757194.670309] [] dqget+0x3fa/0x450 [1757194.675412] [] dquot_get_dqblk+0x14/0x1f0 [1757194.681288] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757194.689011] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757194.695614] [] ofd_quotactl+0x13c/0x380 [ofd] [1757194.701846] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757194.709027] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757194.716945] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757194.723471] [] kthread+0xd1/0xe0 [1757194.728565] [] ret_from_fork_nospec_begin+0xe/0x21 [1757194.735222] [] 0xffffffffffffffff [1757194.740433] LustreError: dumping log to /tmp/lustre-log.1586320618.49112 [1757194.747988] Pid: 50854, comm: ldlm_cn02_031 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757194.758691] Call Trace: [1757194.761340] [] __wait_on_freeing_inode+0xb0/0xf0 [1757194.767837] [] find_inode_fast+0x71/0xb0 [1757194.773623] [] iget_locked+0x79/0x1a0 [1757194.779154] [] ldiskfs_iget+0x42/0xc50 [ldiskfs] [1757194.785688] [] osd_iget+0x26/0x2d0 [osd_ldiskfs] [1757194.792204] [] osd_obj_map_lookup+0x260/0x510 [osd_ldiskfs] [1757194.799649] [] osd_oi_lookup+0x139/0x1e0 [osd_ldiskfs] [1757194.806661] [] osd_fid_lookup+0x455/0x1d60 [osd_ldiskfs] [1757194.813846] [] osd_object_init+0x61/0x110 [osd_ldiskfs] [1757194.820946] [] lu_object_start.isra.35+0x8b/0x120 [obdclass] [1757194.828501] [] lu_object_find_at+0x1e1/0xa60 [obdclass] [1757194.835607] [] lu_object_find+0x16/0x20 [obdclass] [1757194.842281] [] ofd_object_find+0x35/0x100 [ofd] [1757194.848684] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1757194.855176] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1757194.862311] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1757194.869338] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1757194.876459] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757194.884373] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757194.890920] [] kthread+0xd1/0xe0 [1757194.896063] [] ret_from_fork_nospec_begin+0xe/0x21 [1757194.902758] [] 0xffffffffffffffff [1757202.806193] Pid: 95479, comm: ll_ost00_097 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757202.816801] Call Trace: [1757202.819467] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757202.826551] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757202.833804] [] start_this_handle+0x1a1/0x430 [jbd2] [1757202.840537] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757202.847374] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757202.854986] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757202.862162] [] dqget+0x3fa/0x450 [1757202.867246] [] dquot_get_dqblk+0x14/0x1f0 [1757202.873106] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757202.880806] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757202.887372] [] ofd_quotactl+0x13c/0x380 [ofd] [1757202.893584] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757202.900726] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757202.908622] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757202.915130] [] kthread+0xd1/0xe0 [1757202.920219] [] ret_from_fork_nospec_begin+0xe/0x21 [1757202.926867] [] 0xffffffffffffffff [1757202.932049] LustreError: dumping log to /tmp/lustre-log.1586320626.95479 [1757202.944212] Lustre: 49376:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/5), not sending early reply req@ffff9fedcbe2b050 x1659634374955392/t0(0) o3->fc9d29e8-fd99-4@10.50.4.8@o2ib2:701/0 lens 488/0 e 0 to 0 dl 1586320631 ref 2 fl New:/2/ffffffff rc 0/-1 [1757202.971233] Lustre: 49376:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 2451 previous similar messages [1757210.998180] LustreError: dumping log to /tmp/lustre-log.1586320634.49035 [1757227.382136] LustreError: dumping log to /tmp/lustre-log.1586320651.48959 [1757235.574117] LustreError: dumping log to /tmp/lustre-log.1586320659.48827 [1757256.054088] LustreError: dumping log to /tmp/lustre-log.1586320680.95400 [1757258.003682] Lustre: fir-OST0032: Client a251d3b1-58ac-4 (at 10.50.13.2@o2ib2) reconnecting [1757258.012122] Lustre: Skipped 1403 previous similar messages [1757258.017829] Lustre: fir-OST0032: Connection restored to b8ff2e89-6825-4 (at 10.50.13.2@o2ib2) [1757258.026523] Lustre: Skipped 1406 previous similar messages [1757268.342135] LustreError: dumping log to /tmp/lustre-log.1586320692.95550 [1757269.577946] Lustre: fir-OST0032: deleting orphan objects from 0xf80000402:8704421 to 0xf80000402:8704449 [1757280.630048] LustreError: dumping log to /tmp/lustre-log.1586320704.48465 [1757350.261896] LustreError: dumping log to /tmp/lustre-log.1586320774.49057 [1757379.365121] Lustre: fir-OST0030: deleting orphan objects from 0xf00000401:8838661 to 0xf00000401:8838689 [1757383.029826] LustreError: dumping log to /tmp/lustre-log.1586320807.112658 [1757399.413794] LustreError: dumping log to /tmp/lustre-log.1586320823.49209 [1757411.701772] LustreError: dumping log to /tmp/lustre-log.1586320835.95504 [1757436.277723] LustreError: dumping log to /tmp/lustre-log.1586320860.95568 [1757456.757692] LustreError: dumping log to /tmp/lustre-log.1586320880.95661 [1757567.349468] LNet: Service thread pid 95402 was inactive for 1201.63s. The thread might be hung, or it might only be slow and will resume later. Dumping the stack trace for debugging purposes: [1757567.366661] LNet: Skipped 9 previous similar messages [1757567.371899] Pid: 95402, comm: ll_ost00_092 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757567.382518] Call Trace: [1757567.385175] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757567.392259] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757567.399529] [] start_this_handle+0x1a1/0x430 [jbd2] [1757567.406264] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757567.413100] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757567.420712] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757567.427893] [] dqget+0x3fa/0x450 [1757567.432983] [] dquot_get_dqblk+0x14/0x1f0 [1757567.438842] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757567.446563] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757567.453124] [] ofd_quotactl+0x13c/0x380 [ofd] [1757567.459354] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757567.466488] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757567.474391] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757567.480891] [] kthread+0xd1/0xe0 [1757567.485995] [] ret_from_fork_nospec_begin+0xe/0x21 [1757567.492656] [] 0xffffffffffffffff [1757567.497854] LustreError: dumping log to /tmp/lustre-log.1586320991.95402 [1757620.597352] Pid: 95265, comm: ll_ost00_085 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757620.607955] Call Trace: [1757620.610635] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757620.617723] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757620.625017] [] start_this_handle+0x1a1/0x430 [jbd2] [1757620.631753] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757620.638588] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757620.646225] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757620.653408] [] dqget+0x3fa/0x450 [1757620.658497] [] dquot_get_dqblk+0x14/0x1f0 [1757620.664355] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757620.672052] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757620.678614] [] ofd_quotactl+0x13c/0x380 [ofd] [1757620.684826] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757620.691986] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757620.699902] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757620.706422] [] kthread+0xd1/0xe0 [1757620.711512] [] ret_from_fork_nospec_begin+0xe/0x21 [1757620.718160] [] 0xffffffffffffffff [1757620.723344] LustreError: dumping log to /tmp/lustre-log.1586321044.95265 [1757624.693340] Pid: 95659, comm: ll_ost00_103 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757624.703945] Call Trace: [1757624.706600] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757624.713684] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757624.720937] [] start_this_handle+0x1a1/0x430 [jbd2] [1757624.727672] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757624.734492] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757624.742103] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757624.749278] [] dqget+0x3fa/0x450 [1757624.754365] [] dquot_get_dqblk+0x14/0x1f0 [1757624.760224] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757624.767973] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757624.774531] [] ofd_quotactl+0x13c/0x380 [ofd] [1757624.780786] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757624.787913] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757624.795799] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757624.802300] [] kthread+0xd1/0xe0 [1757624.807386] [] ret_from_fork_nospec_begin+0xe/0x21 [1757624.814070] [] 0xffffffffffffffff [1757624.819246] LustreError: dumping log to /tmp/lustre-log.1586321048.95659 [1757628.789336] Pid: 95707, comm: ll_ost00_105 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757628.799942] Call Trace: [1757628.802599] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757628.809682] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757628.816954] [] start_this_handle+0x1a1/0x430 [jbd2] [1757628.823688] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757628.830521] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757628.838134] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757628.845328] [] dqget+0x3fa/0x450 [1757628.850417] [] dquot_get_dqblk+0x14/0x1f0 [1757628.856295] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757628.864020] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757628.870606] [] ofd_quotactl+0x13c/0x380 [ofd] [1757628.876836] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757628.884006] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757628.891922] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757628.898429] [] kthread+0xd1/0xe0 [1757628.903550] [] ret_from_fork_nospec_begin+0xe/0x21 [1757628.910199] [] 0xffffffffffffffff [1757628.915417] LustreError: dumping log to /tmp/lustre-log.1586321052.95707 [1757686.133219] Pid: 112659, comm: ll_ost01_102 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757686.143909] Call Trace: [1757686.146567] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757686.153650] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757686.160921] [] start_this_handle+0x1a1/0x430 [jbd2] [1757686.167672] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757686.174509] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757686.182134] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757686.189305] [] dqget+0x3fa/0x450 [1757686.194391] [] dquot_get_dqblk+0x14/0x1f0 [1757686.200253] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757686.207965] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757686.214540] [] ofd_quotactl+0x13c/0x380 [ofd] [1757686.220754] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757686.227889] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757686.235784] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757686.242284] [] kthread+0xd1/0xe0 [1757686.247396] [] ret_from_fork_nospec_begin+0xe/0x21 [1757686.254043] [] 0xffffffffffffffff [1757686.259240] LustreError: dumping log to /tmp/lustre-log.1586321110.112659 [1757710.709168] LNet: Service thread pid 95404 was inactive for 1202.08s. Watchdog stack traces are limited to 3 per 300 seconds, skipping this one. [1757710.722294] LNet: Skipped 32 previous similar messages [1757710.727615] LustreError: dumping log to /tmp/lustre-log.1586321134.95404 [1757714.805152] LustreError: dumping log to /tmp/lustre-log.1586321138.95783 [1757722.997134] LustreError: dumping log to /tmp/lustre-log.1586321146.95723 [1757755.765059] LustreError: dumping log to /tmp/lustre-log.1586321179.48842 [1757776.245026] LustreError: dumping log to /tmp/lustre-log.1586321200.48950 [1757780.341007] LustreError: dumping log to /tmp/lustre-log.1586321204.48311 [1757796.724979] LustreError: dumping log to /tmp/lustre-log.1586321220.48960 [1757803.456998] Lustre: 95535:0:(service.c:1372:ptlrpc_at_send_early_reply()) @@@ Couldn't add any time (5/-150), not sending early reply req@ffff9fe57d2be050 x1659460164104256/t0(0) o17->39bbae28-5b70-4@10.50.1.35@o2ib2:547/0 lens 456/0 e 0 to 0 dl 1586321232 ref 2 fl New:/0/ffffffff rc 0/-1 [1757803.484478] Lustre: 95535:0:(service.c:1372:ptlrpc_at_send_early_reply()) Skipped 3893 previous similar messages [1757804.916961] LustreError: dumping log to /tmp/lustre-log.1586321228.95843 [1757817.204937] LustreError: dumping log to /tmp/lustre-log.1586321241.95533 [1757858.061189] Lustre: fir-OST0034: Client 91b25f9a-b6af-4 (at 10.50.1.38@o2ib2) reconnecting [1757858.069633] Lustre: Skipped 2191 previous similar messages [1757858.075325] Lustre: fir-OST0034: Connection restored to 91b25f9a-b6af-4 (at 10.50.1.38@o2ib2) [1757858.084063] Lustre: Skipped 2192 previous similar messages [1757858.164850] LustreError: dumping log to /tmp/lustre-log.1586321282.95599 [1757874.548812] Pid: 95844, comm: ll_ost03_104 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757874.559414] Call Trace: [1757874.562072] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757874.569156] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757874.576408] [] start_this_handle+0x1a1/0x430 [jbd2] [1757874.583143] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757874.589963] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757874.597572] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757874.604748] [] dqget+0x3fa/0x450 [1757874.609837] [] dquot_get_dqblk+0x14/0x1f0 [1757874.615694] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757874.623399] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757874.629959] [] ofd_quotactl+0x13c/0x380 [ofd] [1757874.636174] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757874.643298] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757874.651183] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757874.657684] [] kthread+0xd1/0xe0 [1757874.662771] [] ret_from_fork_nospec_begin+0xe/0x21 [1757874.669420] [] 0xffffffffffffffff [1757874.674606] LustreError: dumping log to /tmp/lustre-log.1586321298.95844 [1757890.932774] Pid: 50864, comm: ldlm_cn03_024 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757890.943468] Call Trace: [1757890.946117] [] lu_object_find_at+0x847/0xa60 [obdclass] [1757890.953215] [] lu_object_find+0x16/0x20 [obdclass] [1757890.959890] [] ofd_object_find+0x35/0x100 [ofd] [1757890.966294] [] ofd_lvbo_update+0x56f/0xe40 [ofd] [1757890.972800] [] ldlm_request_cancel+0x404/0x780 [ptlrpc] [1757890.979928] [] ldlm_handle_cancel+0x232/0x2b0 [ptlrpc] [1757890.986961] [] ldlm_cancel_handler+0x158/0x590 [ptlrpc] [1757890.994082] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757891.001975] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757891.008477] [] kthread+0xd1/0xe0 [1757891.013583] [] ret_from_fork_nospec_begin+0xe/0x21 [1757891.020233] [] 0xffffffffffffffff [1757891.025463] LustreError: dumping log to /tmp/lustre-log.1586321315.50864 [1757911.412730] Pid: 95862, comm: ll_ost03_105 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757911.423337] Call Trace: [1757911.426000] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757911.433082] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757911.440380] [] start_this_handle+0x1a1/0x430 [jbd2] [1757911.447114] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757911.453934] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757911.461545] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757911.468713] [] dqget+0x3fa/0x450 [1757911.473799] [] dquot_get_dqblk+0x14/0x1f0 [1757911.479692] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757911.487388] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757911.493949] [] ofd_quotactl+0x13c/0x380 [ofd] [1757911.500161] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757911.507288] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757911.515173] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757911.521673] [] kthread+0xd1/0xe0 [1757911.526760] [] ret_from_fork_nospec_begin+0xe/0x21 [1757911.533407] [] 0xffffffffffffffff [1757911.538595] LustreError: dumping log to /tmp/lustre-log.1586321335.95862 [1757944.180662] Pid: 95598, comm: ll_ost01_110 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757944.191267] Call Trace: [1757944.193934] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757944.201035] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757944.208312] [] start_this_handle+0x1a1/0x430 [jbd2] [1757944.215068] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757944.221893] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757944.229501] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757944.236684] [] dqget+0x3fa/0x450 [1757944.241773] [] dquot_get_dqblk+0x14/0x1f0 [1757944.247632] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757944.255344] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757944.261924] [] ofd_quotactl+0x13c/0x380 [ofd] [1757944.268154] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757944.275307] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757944.283207] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757944.289735] [] kthread+0xd1/0xe0 [1757944.294850] [] ret_from_fork_nospec_begin+0xe/0x21 [1757944.301509] [] 0xffffffffffffffff [1757944.306726] LustreError: dumping log to /tmp/lustre-log.1586321368.95598 [1757981.044577] Pid: 95908, comm: ll_ost03_111 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 SMP Thu Nov 7 15:26:16 PST 2019 [1757981.055179] Call Trace: [1757981.057833] [] wait_transaction_locked+0x85/0xd0 [jbd2] [1757981.064920] [] add_transaction_credits+0x268/0x2f0 [jbd2] [1757981.072172] [] start_this_handle+0x1a1/0x430 [jbd2] [1757981.078908] [] jbd2__journal_start+0xf3/0x1f0 [jbd2] [1757981.085773] [] __ldiskfs_journal_start_sb+0x69/0xe0 [ldiskfs] [1757981.093388] [] ldiskfs_acquire_dquot+0x53/0xb0 [ldiskfs] [1757981.100565] [] dqget+0x3fa/0x450 [1757981.105686] [] dquot_get_dqblk+0x14/0x1f0 [1757981.111546] [] osd_acct_index_lookup+0x235/0x480 [osd_ldiskfs] [1757981.119241] [] lquotactl_slv+0x27d/0xa00 [lquota] [1757981.125802] [] ofd_quotactl+0x13c/0x380 [ofd] [1757981.132015] [] tgt_request_handle+0xada/0x1570 [ptlrpc] [1757981.139177] [] ptlrpc_server_handle_request+0x24b/0xab0 [ptlrpc] [1757981.147060] [] ptlrpc_main+0xb34/0x1470 [ptlrpc] [1757981.153597] [] kthread+0xd1/0xe0 [1757981.158715] [] ret_from_fork_nospec_begin+0xe/0x21 [1757981.165365] [] 0xffffffffffffffff [1757981.170586] LustreError: dumping log to /tmp/lustre-log.1586321405.95908 [1758005.620530] LustreError: dumping log to /tmp/lustre-log.1586321429.49457 [1758022.004498] LustreError: dumping log to /tmp/lustre-log.1586321446.48272 [1758038.388452] LustreError: dumping log to /tmp/lustre-log.1586321462.95907 [1758054.772425] LustreError: dumping log to /tmp/lustre-log.1586321478.95882 [1758059.486889] SysRq : Trigger a crash [1758059.490629] BUG: unable to handle kernel NULL pointer dereference at (null) [1758059.498681] IP: [] sysrq_handle_crash+0x16/0x20 [1758059.504973] PGD 3a8616d067 PUD 3e5ebd9067 PMD 0 [1758059.509844] Oops: 0002 [#1] SMP [1758059.513302] Modules linked in: osp(OE) ofd(OE) lfsck(OE) ost(OE) mgc(OE) osd_ldiskfs(OE) lquota(OE) raid456 async_raid6_recov async_memcpy async_pq raid6_pq libcrc32c async_xor xor async_tx ldiskfs(OE) lmv(OE) osc(OE) lov(OE) fid(OE) fld(OE) ko2iblnd(OE) ptlrpc(OE) obdclass(OE) lnet(OE) libcfs(OE) rpcsec_gss_krb5 auth_rpcgss nfsv4 dns_resolver nfs lockd grace fscache rdma_ucm(OE) ib_ucm(OE) rdma_cm(OE) iw_cm(OE) ib_ipoib(OE) ib_cm(OE) ib_umad(OE) mlx4_en(OE) mlx4_ib(OE) mlx4_core(OE) dell_rbu sunrpc vfat fat dcdbas amd64_edac_mod edac_mce_amd kvm_amd kvm irqbypass crc32_pclmul ghash_clmulni_intel aesni_intel lrw gf128mul glue_helper ablk_helper cryptd pcspkr dm_service_time dm_multipath ses enclosure ipmi_si ipmi_devintf dm_mod sg k10temp ipmi_msghandler ccp acpi_power_meter i2c_piix4 ip_tables ext4 [1758059.586014] mbcache jbd2 sd_mod crc_t10dif crct10dif_generic mlx5_ib(OE) ib_uverbs(OE) ib_core(OE) i2c_algo_bit drm_kms_helper syscopyarea sysfillrect sysimgblt fb_sys_fops mlx5_core(OE) ttm ahci mlxfw(OE) devlink mpt3sas(OE) libahci tg3 drm crct10dif_pclmul crct10dif_common mlx_compat(OE) libata crc32c_intel raid_class ptp megaraid_sas scsi_transport_sas drm_panel_orientation_quirks pps_core [last unloaded: mdc] [1758059.622227] CPU: 23 PID: 96554 Comm: bash Kdump: loaded Tainted: G OE ------------ 3.10.0-957.27.2.el7_lustre.pl2.x86_64 #1 [1758059.634643] Hardware name: Dell Inc. PowerEdge R6415/07YXFK, BIOS 1.10.6 08/15/2019 [1758059.642470] task: ffff9ff7679c4100 ti: ffff9ff9e4e74000 task.ti: ffff9ff9e4e74000 [1758059.650122] RIP: 0010:[] [] sysrq_handle_crash+0x16/0x20 [1758059.658842] RSP: 0018:ffff9ff9e4e77e58 EFLAGS: 00010246 [1758059.664327] RAX: ffffffff9e664430 RBX: ffffffff9eee4f80 RCX: 0000000000000000 [1758059.671632] RDX: 0000000000000000 RSI: ffffa00c3f553898 RDI: 0000000000000063 [1758059.678938] RBP: ffff9ff9e4e77e58 R08: ffffffff9f1e38bc R09: ffffffff9f220dc3 [1758059.686245] R10: 00000000000031c0 R11: 00000000000031bf R12: 0000000000000063 [1758059.693551] R13: 0000000000000000 R14: 0000000000000007 R15: 0000000000000000 [1758059.700858] FS: 00007f23b763f740(0000) GS:ffffa00c3f540000(0000) knlGS:0000000000000000 [1758059.709115] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [1758059.715036] CR2: 0000000000000000 CR3: 0000003bdee9a000 CR4: 00000000003407e0 [1758059.722343] Call Trace: [1758059.724970] [] __handle_sysrq+0x10d/0x170 [1758059.730803] [] write_sysrq_trigger+0x28/0x40 [1758059.736897] [] proc_reg_write+0x40/0x80 [1758059.742555] [] vfs_write+0xc0/0x1f0 [1758059.747865] [] SyS_write+0x7f/0xf0 [1758059.753095] [] system_call_fastpath+0x22/0x27 [1758059.759269] Code: eb 9b 45 01 f4 45 39 65 34 75 e5 4c 89 ef e8 e2 f7 ff ff eb db 66 66 66 66 90 55 48 89 e5 c7 05 91 31 7e 00 01 00 00 00 0f ae f8 04 25 00 00 00 00 01 5d c3 66 66 66 66 90 55 31 c0 c7 05 0e [1758059.779965] RIP [] sysrq_handle_crash+0x16/0x20 [1758059.786343] RSP [1758059.790009] CR2: 0000000000000000