ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x4/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000282 <4>RAX: ffff8807fc043000 RBX: ffff8806db06bd00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 8f 90 02 00 00 31 d2 0f b7 46 10 48 83 39 3f 76 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 <0f> 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 25992:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880bdcf6c000 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3b3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff8806db06bdcf RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 48 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 <48> 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x299/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2552b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 47 18 41 8b 57 14 83 c0 01 41 89 47 18 01 d0 41 83 fe 02 89 83 34 14 00 00 41 0f 94 c6 4d 85 ed 74 09 4d 85 e4 0f 85 d7 00 00 00 <45> 84 f6 0f 85 a9 00 00 00 f6 83 3c 14 00 00 04 0f 85 9c 00 00 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 24600:0:(service.c:1290:ptlrpc_at_send_early_reply()) @@@ Already past deadline (-945s), not sending early reply. Consider increasing at_early_margin (5)? req@ffff881003986850 x1576937591629312/t0(0) o103->34235682-d281-7d02-8263-f5f018a9d20f@1283@gni5:661/0 lens 328/224 e 0 to 0 dl 1504699806 ref 2 fl Interpret:/0/0 rc 0/0 <4>Lustre: 24367:0:(service.c:2097:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (342:1109s); client may timeout. req@ffff8806a69c4c50 x1576939301787264/t0(0) o101->8dd26703-c5c0-7c70-b914-adcd2169000c@698@gni5:497/0 lens 328/368 e 1 to 0 dl 1504699642 ref 1 fl Complete:/0/0 rc 0/0 <4>Lustre: 24367:0:(service.c:2097:ptlrpc_server_handle_request()) Skipped 293 previous similar messages <4>LNet: Service thread pid 24367 completed after 1450.70s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <4>Lustre: 24367:0:(service.c:1945:ptlrpc_server_handle_req_in()) @@@ Slow req_in handling 1118s req@ffff8804d0de2450 x1576937469134032/t0(0) o400->9939b31f-6ff8-be49-388e-c546d50935f6@233@gni5:0/0 lens 224/0 e 0 to 0 dl 0 ref 1 fl New:/0/ffffffff rc 0/-1 <4>Lustre: 24367:0:(service.c:1945:ptlrpc_server_handle_req_in()) Skipped 44 previous similar messages <4>Lustre: ost: This server is not able to keep up with request traffic (cpu-bound). <4>Lustre: 24367:0:(service.c:1502:ptlrpc_at_check_timed()) earlyQ=274 reqQ=115316 recA=121, svcEst=600, delay=1115090(jiff) <3>LustreError: 24367:0:(service.c:2059:ptlrpc_server_handle_request()) @@@ Dropping timed-out request from 12345-233@gni5: deadline 432:686s ago <3> req@ffff8804d0de2450 x1576937469134032/t0(0) o400->9939b31f-6ff8-be49-388e-c546d50935f6@233@gni5:165/0 lens 224/0 e 0 to 0 dl 1504700065 ref 1 fl Interpret:H/0/ffffffff rc 0/-1 <3>LustreError: 24367:0:(service.c:2059:ptlrpc_server_handle_request()) Skipped 235 previous similar messages <4>Lustre: 24600:0:(service.c:1290:ptlrpc_at_send_early_reply()) Skipped 4026 previous similar messages <3>LustreError: 29983:0:(ldlm_lib.c:3177:target_bulk_io()) @@@ timeout on bulk WRITE after -1140+1140s req@ffff8807e54e0680 x1573094892999800/t0(0) o4->adbb815b-a33a-c229-209d-e0f634a1537a@1075@gni4:467/0 lens 488/448 e 0 to 0 dl 1504699612 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: f1-OST006f: Bulk IO write error with 713ef061-ea83-55b9-5ec2-a677ea4f842b (at 709@gni4), client will retry: rc -110 <4>Lustre: Skipped 4 previous similar messages <4>LNet: Service thread pid 25964 completed after 1422.00s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <3>LustreError: 29983:0:(ldlm_lib.c:3177:target_bulk_io()) Skipped 1 previous similar message <3>LustreError: 25972:0:(ldlm_lib.c:3177:target_bulk_io()) @@@ timeout on bulk WRITE after -1141+1141s req@ffff880346403c80 x1573094469095320/t0(0) o4->2ccce0cf-add4-ec03-870a-e0d64bfdd780@222@gni4:468/0 lens 488/448 e 0 to 0 dl 1504699613 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: f1-OST00bf: Bulk IO write error with 2ccce0cf-add4-ec03-870a-e0d64bfdd780 (at 222@gni4), client will retry: rc -110 <4>Lustre: Skipped 1 previous similar message <4>LNet: Service thread pid 25972 completed after 1458.44s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <4>LNet: Skipped 3 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: ldlm_canceld: This server is not able to keep up with request traffic (cpu-bound). <3>LustreError: 24463:0:(ldlm_lib.c:3177:target_bulk_io()) @@@ timeout on bulk WRITE after -1272+1272s req@ffff8801671bb0c0 x1573094900119832/t0(0) o4->ce0a1c7b-ecaa-1760-404b-5a1dbcb813df@292@gni4:467/0 lens 488/448 e 0 to 0 dl 1504699612 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: f1-OST0097: Bulk IO write error with ce0a1c7b-ecaa-1760-404b-5a1dbcb813df (at 292@gni4), client will retry: rc -110 <4>Lustre: 24463:0:(service.c:2097:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (317:1272s); client may timeout. req@ffff8801671bb0c0 x1573094900119832/t0(0) o4->ce0a1c7b-ecaa-1760-404b-5a1dbcb813df@292@gni4:467/0 lens 488/448 e 0 to 0 dl 1504699612 ref 1 fl Complete:/0/ffffffff rc -110/-1 <4>Lustre: 24463:0:(service.c:2097:ptlrpc_server_handle_request()) Skipped 599 previous similar messages <4>LNet: Service thread pid 24463 completed after 1589.73s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <4>Lustre: 24287:0:(service.c:1945:ptlrpc_server_handle_req_in()) @@@ Slow req_in handling 1222s req@ffff880ca4fcb050 x1576937693964448/t0(0) o400->3f2d8192-3e84-9742-8468-176e39eb9df3@914@gni5:0/0 lens 224/0 e 0 to 0 dl 0 ref 1 fl New:/0/ffffffff rc 0/-1 <4>Lustre: 24287:0:(service.c:1945:ptlrpc_server_handle_req_in()) Skipped 33957 previous similar messages <4>Lustre: f1-OST006f: haven't heard from client f19720df-e812-b7c6-5ec0-319c426bb399 (at 10.10.32.134@o2ib2) in 1591 seconds. I think it's dead, and I am evicting it. exp ffff880ff7825c00, cur 1504700885 expire 1504699985 last 1504699294 <4>Lustre: 24287:0:(service.c:1502:ptlrpc_at_check_timed()) earlyQ=263 reqQ=117605 recA=121, svcEst=600, delay=1220431(jiff) <4>Lustre: 24287:0:(service.c:1502:ptlrpc_at_check_timed()) Skipped 339 previous similar messages <4>Lustre: 24287:0:(service.c:1290:ptlrpc_at_send_early_reply()) @@@ Already past deadline (-759s), not sending early reply. Consider increasing at_early_margin (5)? req@ffff880ca5e8d850 x1576938124593632/t0(0) o400->58252420-590d-8a01-1c36-4f075a77da2b@288@gni5:226/0 lens 224/0 e 0 to 0 dl 1504700126 ref 2 fl New:H/0/ffffffff rc 0/-1 <4>Lustre: 24287:0:(service.c:1290:ptlrpc_at_send_early_reply()) Skipped 30036 previous similar messages <3>LustreError: 24287:0:(service.c:2059:ptlrpc_server_handle_request()) @@@ Dropping timed-out request from 12345-914@gni5: deadline 317:905s ago <3> req@ffff880ca4fcb050 x1576937693964448/t0(0) o400->3f2d8192-3e84-9742-8468-176e39eb9df3@914@gni5:80/0 lens 224/0 e 0 to 0 dl 1504699980 ref 1 fl Interpret:H/0/ffffffff rc 0/-1 <3>LustreError: 24287:0:(service.c:2059:ptlrpc_server_handle_request()) Skipped 593 previous similar messages <4>Lustre: Skipped 340 previous similar messages <3>LustreError: 24975:0:(ldlm_lib.c:3137:target_bulk_io()) @@@ bulk WRITE failed: rc -107 req@ffff880446406cc0 x1573095506394368/t0(0) o4->10532c75-41d3-6d03-340c-7bab51c6c446@323@gni4:473/0 lens 488/448 e 0 to 0 dl 1504699618 ref 1 fl Interpret:/0/0 rc 0/0 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x23/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 3f 76 07 0f b7 56 32 c1 e2 10 <0f> b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <3>LustreError: 24976:0:(ldlm_lib.c:3177:target_bulk_io()) @@@ timeout on bulk WRITE after -1259+1259s req@ffff8806b55116c0 x1573094772525272/t0(0) o4->01224b17-e03b-8f59-6393-01fddfe93f79@1272@gni4:499/0 lens 488/448 e 0 to 0 dl 1504699644 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: 64833:0:(service.c:2097:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (317:879s); client may timeout. req@ffff88071c91a9c0 x1576937839075024/t0(0) o103->2a4bc8fd-89f1-f65c-b7bf-23f9acb8ee02@108@gni5:124/0 lens 336/192 e 0 to 0 dl 1504700024 ref 1 fl Complete:/0/0 rc 0/0 <4>Lustre: 64833:0:(service.c:2097:ptlrpc_server_handle_request()) Skipped 207 previous similar messages <3>LustreError: 24976:0:(ldlm_lib.c:3177:target_bulk_io()) Skipped 3 previous similar messages <4>Lustre: f1-OST010f: Bulk IO write error with 01224b17-e03b-8f59-6393-01fddfe93f79 (at 1272@gni4), client will retry: rc -110 <4>Lustre: Skipped 3 previous similar messages <4>LNet: Service thread pid 24976 completed after 1559.28s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <4>LNet: Skipped 13 previous similar messages <4>Lustre: 114733:0:(service.c:1290:ptlrpc_at_send_early_reply()) @@@ Already past deadline (-1191s), not sending early reply. Consider increasing at_early_margin (5)? req@ffff8808e8e590c0 x1576943343923792/t0(0) o103->294190b5-a66a-e535-9f20-82820c705433@129@gni5:568/0 lens 328/224 e 1 to 0 dl 1504699713 ref 2 fl Interpret:/0/0 rc 0/0 <4>Lustre: ldlm_canceld: This server is not able to keep up with request traffic (cpu-bound). <4>Lustre: Skipped 36 previous similar messages <4>Lustre: 64833:0:(service.c:1502:ptlrpc_at_check_timed()) earlyQ=1 reqQ=0 recA=57, svcEst=600, delay=0(jiff) <4>Lustre: 64833:0:(service.c:1502:ptlrpc_at_check_timed()) Skipped 37 previous similar messages <4>Lustre: 24259:0:(service.c:1945:ptlrpc_server_handle_req_in()) @@@ Slow req_in handling 1089s req@ffff8805b7707450 x1573094587955872/t0(0) o400->7aad12dc-c275-e09b-57a4-183fe86221b9@1204@gni4:0/0 lens 224/0 e 0 to 0 dl 0 ref 1 fl New:/0/ffffffff rc 0/-1 <4>Lustre: 24259:0:(service.c:1945:ptlrpc_server_handle_req_in()) Skipped 3378 previous similar messages <3>LustreError: 24259:0:(service.c:2059:ptlrpc_server_handle_request()) @@@ Dropping timed-out request from 12345-758@gni4: deadline 317:952s ago <3> req@ffff88059bb15850 x1573094104504020/t0(0) o400->58c02606-b9ea-107d-9df5-5f4c0c75e2c7@758@gni4:52/0 lens 224/0 e 0 to 0 dl 1504699952 ref 1 fl Interpret:H/0/ffffffff rc 0/-1 <3>LustreError: 24259:0:(service.c:2059:ptlrpc_server_handle_request()) Skipped 193 previous similar messages <4>Lustre: 114733:0:(service.c:1290:ptlrpc_at_send_early_reply()) Skipped 4867 previous similar messages <3>LustreError: 26003:0:(ldlm_lib.c:3137:target_bulk_io()) @@@ bulk WRITE failed: rc -107 req@ffff880324db2080 x1573094386458796/t0(0) o4->380473b9-bf4e-0001-043c-9b93ade30e29@553@gni4:467/0 lens 488/448 e 0 to 0 dl 1504699612 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: 25992:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880bdcf6c000 <4>Lustre: 25992:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 4 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3a3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f7 71 18 89 55 b8 48 8b 87 90 02 00 00 8b 40 18 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 <48> 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <3>LustreError: 24468:0:(ldlm_lockd.c:342:waiting_locks_callback()) ### lock callback timer expired after 376s: evicting client at 10.10.32.104@o2ib2 ns: filter-f1-OST00bf_UUID lock: ffff880f1049c0c0/0x972469791f7097fb lrc: 4/0,0 mode: PR/PR res: [0xa2ec79a:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x60000400010020 nid: 10.10.32.104@o2ib2 remote: 0xcc3bb3cf96fc2a14 expref: 283 pid: 24371 timeout: 4328698061 lvb_type: 1 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x361/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f25577 <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 00 39 83 34 14 00 00 0f 82 8c 00 00 00 48 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 <4c> 8b 7d f8 c9 c3 66 0f 1f 84 00 00 00 00 00 41 f6 85 18 0a 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>LNet: 16708:0:(o2iblnd_cb.c:397:kiblnd_handle_rx()) PUT_NACK from 10.10.32.102@o2ib2 <3>LustreError: 29985:0:(ldlm_lib.c:3177:target_bulk_io()) @@@ timeout on bulk READ after -1521+1521s req@ffff88026bd256c0 x1576651845774900/t0(0) o3->7f40e496-c123-c6a1-7eaa-837a3f29d4e2@10.10.32.102@o2ib2:499/0 lens 488/432 e 0 to 0 dl 1504699644 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: 24969:0:(service.c:2097:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (317:1547s); client may timeout. req@ffff88044de7acc0 x1573094634085228/t150324585918(0) o4->92837cde-441a-689e-795a-62b88291e540@856@gni4:473/0 lens 488/416 e 0 to 0 dl 1504699618 ref 1 fl Complete:/0/0 rc 0/0 <4>Lustre: 24969:0:(service.c:2097:ptlrpc_server_handle_request()) Skipped 19 previous similar messages <4>LNet: Service thread pid 24969 completed after 1864.10s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <4>LNet: Skipped 4 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 25992:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880bdcf6c000 <4>Lustre: 25992:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 4 previous similar messages <3>LustreError: 59942:0:(ldlm_lockd.c:342:waiting_locks_callback()) ### lock callback timer expired after 376s: evicting client at 10.10.32.235@o2ib2 ns: filter-f1-OST006f_UUID lock: ffff880bebb2b300/0x972469791f56ce92 lrc: 3/0,0 mode: PR/PR res: [0xa0d92de:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x60000400010020 nid: 10.10.32.235@o2ib2 remote: 0x9d0348a7207cd3ea expref: 15 pid: 24284 timeout: 4328831224 lvb_type: 1 <3>LustreError: 59942:0:(ldlm_lockd.c:342:waiting_locks_callback()) Skipped 1 previous similar message <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <3>LustreError: 24479:0:(ldlm_lib.c:3137:target_bulk_io()) @@@ bulk WRITE failed: rc -107 req@ffff8803db36d6c0 x1573094563460684/t0(0) o4->d2cf33dd-096c-a3f2-3140-2854efe798a4@440@gni4:469/0 lens 488/448 e 0 to 0 dl 1504699614 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: f1-OST001f: Bulk IO write error with d2cf33dd-096c-a3f2-3140-2854efe798a4 (at 440@gni4), client will retry: rc -107 <4>Lustre: Skipped 1 previous similar message <4>Lustre: 24479:0:(service.c:2097:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (317:1703s); client may timeout. req@ffff8803db36d6c0 x1573094563460684/t0(0) o4->d2cf33dd-096c-a3f2-3140-2854efe798a4@440@gni4:469/0 lens 488/448 e 0 to 0 dl 1504699614 ref 1 fl Complete:/0/ffffffff rc -107/-1 <4>LNet: Service thread pid 24479 completed after 2017.10s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <4>Lustre: 17873:0:(client.c:2063:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1504700752/real 1504700752] req@ffff8804bfc870c0 x1577758166457312/t0(0) o104->f1-OST00bf@10.10.32.104@o2ib2:15/16 lens 296/224 e 0 to 1 dl 1504701319 ref 1 fl Rpc:X/0/ffffffff rc 0/-1 <3>LustreError: 17873:0:(ldlm_lockd.c:689:ldlm_handle_ast_error()) ### client (nid 10.10.32.104@o2ib2) failed to reply to blocking AST (req status 0 rc -110), evict it ns: filter-f1-OST00bf_UUID lock: ffff880f1049c0c0/0x972469791f7097fb lrc: 4/0,0 mode: PR/PR res: [0xa2ec79a:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x60000400010020 nid: 10.10.32.104@o2ib2 remote: 0xcc3bb3cf96fc2a14 expref: 283 pid: 24371 timeout: 4328698061 lvb_type: 1 <3>LustreError: 138-a: f1-OST00bf: A client on nid 10.10.32.104@o2ib2 was evicted due to a lock blocking callback time out: rc -110 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x40/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: ffff8807fdaf0fd8 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff02 ffffffffa0f2552b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 45 31 e4 48 8d 86 d8 0f 00 00 48 89 7d 88 48 89 4d 98 4c 89 45 90 48 89 f3 49 89 d7 <48> 89 85 78 ff ff ff 4c 8b ae d0 0f 00 00 45 89 ce 49 8d 45 08 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0x1b/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000286 <4>RAX: ffff8807fc043400 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: ffff8807fdaf0fd8 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06bd00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 ffff8806db06bdcf 0000000000000004 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 89 e5 48 81 ec a0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 6d e8 4c 89 75 f0 <4c> 89 7d f8 0f 1f 44 00 00 48 8b 05 d5 ce 73 ff 49 89 ff 48 89 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 24348:0:(service.c:2097:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (392:1833s); client may timeout. req@ffff880f889a1450 x1576938909833840/t0(0) o101->c6e6c6df-3107-30e5-2cff-a48f702816ab@1182@gni5:547/0 lens 328/368 e 3 to 0 dl 1504699692 ref 1 fl Complete:/0/0 rc 0/0 <4>LNet: Service thread pid 24333 completed after 2224.70s. This indicates the system was overloaded (too many service threads, or there were not enough hardware resources). <4>Lustre: 24333:0:(service.c:1945:ptlrpc_server_handle_req_in()) @@@ Slow req_in handling 1862s req@ffff880cb67c0850 x1576937847260416/t0(0) o400->f167ab1c-5776-f02d-7010-c0d1840329d7@103@gni5:0/0 lens 224/0 e 0 to 0 dl 0 ref 1 fl New:/0/ffffffff rc 0/-1 <4>Lustre: 24333:0:(service.c:1945:ptlrpc_server_handle_req_in()) Skipped 1178 previous similar messages <4>Lustre: ost: This server is not able to keep up with request traffic (cpu-bound). <4>Lustre: Skipped 12 previous similar messages <4>Lustre: 24333:0:(service.c:1502:ptlrpc_at_check_timed()) earlyQ=78 reqQ=144536 recA=120, svcEst=600, delay=640656(jiff) <4>Lustre: 24333:0:(service.c:1502:ptlrpc_at_check_timed()) Skipped 12 previous similar messages <4>Lustre: 24333:0:(service.c:1290:ptlrpc_at_send_early_reply()) @@@ Already past deadline (-1398s), not sending early reply. Consider increasing at_early_margin (5)? req@ffff880cedc37c50 x1573095059118232/t0(0) o400->9b654bf3-2d2d-e020-400f-f79d62123c56@584@gni4:227/0 lens 224/0 e 0 to 0 dl 1504700127 ref 2 fl New:H/0/ffffffff rc 0/-1 <4>Lustre: 24333:0:(service.c:1290:ptlrpc_at_send_early_reply()) Skipped 2 previous similar messages <3>LustreError: 24333:0:(service.c:2059:ptlrpc_server_handle_request()) @@@ Dropping timed-out request from 12345-914@gni5: deadline 317:1545s ago <3> req@ffff880cedc37850 x1576937693965088/t0(0) o400->3f2d8192-3e84-9742-8468-176e39eb9df3@914@gni5:80/0 lens 224/0 e 0 to 0 dl 1504699980 ref 1 fl Interpret:H/0/ffffffff rc 0/-1 <3>LustreError: 24333:0:(service.c:2059:ptlrpc_server_handle_request()) Skipped 11 previous similar messages <3>LustreError: 24485:0:(ldlm_lib.c:3177:target_bulk_io()) @@@ timeout on bulk WRITE after -1913+1913s req@ffff8808707a9080 x1573093678113180/t0(0) o4->d0873d13-11b1-417c-ee2e-2483758aad2d@1654@gni4:467/0 lens 488/448 e 0 to 0 dl 1504699612 ref 1 fl Interpret:/0/0 rc 0/0 <4>Lustre: f1-OST0097: Bulk IO write error with d0873d13-11b1-417c-ee2e-2483758aad2d (at 1654@gni4), client will retry: rc -110 <4>Lustre: 24348:0:(service.c:2097:ptlrpc_server_handle_request()) Skipped 6 previous similar messages <4>Lustre: 24348:0:(service.c:1945:ptlrpc_server_handle_req_in()) @@@ Slow req_in handling 1864s req@ffff880d26241850 x1576937677983632/t0(0) o400->067a1e27-234c-1aeb-9f85-f854403c53a8@1126@gni5:0/0 lens 224/0 e 0 to 0 dl 0 ref 1 fl New:/0/ffffffff rc 0/-1 <4>Lustre: 24348:0:(service.c:1945:ptlrpc_server_handle_req_in()) Skipped 224 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x4/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000282 <4>RAX: ffff8807fc043000 RBX: ffff8806db06bd00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 8f 90 02 00 00 31 d2 0f b7 46 10 48 83 39 3f 76 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 <0f> 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <3>LustreError: 24547:0:(ldlm_request.c:106:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1504700885, 750s ago); not entering recovery in server code, just going back to sleep ns: filter-f1-OST006f_UUID lock: ffff8803c8250280/0x972469791f710497 lrc: 3/0,1 mode: --/PW res: [0xa0d92de:0x0:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 24547 timeout: 0 lvb_type: 0 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: ldlm_canceld: This server is not able to keep up with request traffic (cpu-bound). <4>Lustre: Skipped 1 previous similar message <4>Lustre: 114733:0:(service.c:1502:ptlrpc_at_check_timed()) earlyQ=1 reqQ=0 recA=56, svcEst=600, delay=0(jiff) <4>Lustre: 24333:0:(service.c:1945:ptlrpc_server_handle_req_in()) @@@ Slow req_in handling 2030s req@ffff880d293c6450 x1573095267333812/t0(0) o400->f5c59459-d117-6660-c698-e5ee4ce9470a@566@gni4:0/0 lens 224/0 e 0 to 0 dl 0 ref 1 fl New:/0/ffffffff rc 0/-1 <4>Lustre: 24333:0:(service.c:1945:ptlrpc_server_handle_req_in()) Skipped 3 previous similar messages <4>Lustre: 24333:0:(service.c:1290:ptlrpc_at_send_early_reply()) @@@ Already past deadline (-1567s), not sending early reply. Consider increasing at_early_margin (5)? req@ffff880d7b670850 x1573092643466112/t0(0) o400->ad173c9a-72f9-d1af-2b47-01d160d248db@1687@gni4:229/0 lens 224/0 e 0 to 0 dl 1504700129 ref 2 fl New:H/0/ffffffff rc 0/-1 <4>Lustre: 24333:0:(service.c:1290:ptlrpc_at_send_early_reply()) Skipped 194 previous similar messages <3>LustreError: 24333:0:(service.c:2059:ptlrpc_server_handle_request()) @@@ Dropping timed-out request from 12345-339@gni5: deadline 432:1601s ago <3> req@ffff880cb67bf050 x1576937647755008/t0(0) o400->1f8200ce-47ce-ed24-4fc7-92387977e72e@339@gni5:195/0 lens 224/0 e 0 to 0 dl 1504700095 ref 1 fl Interpret:H/0/ffffffff rc 0/-1 <3>LustreError: 24333:0:(service.c:2059:ptlrpc_server_handle_request()) Skipped 2 previous similar messages <4>Lustre: 24333:0:(service.c:2097:ptlrpc_server_handle_request()) @@@ Request took longer than estimated (432:1601s); client may timeout. req@ffff880cb67bf050 x1576937647755008/t0(0) o400->1f8200ce-47ce-ed24-4fc7-92387977e72e@339@gni5:195/0 lens 224/0 e 0 to 0 dl 1504700095 ref 1 fl Interpret:H/0/ffffffff rc 0/-1 <4>Lustre: 114733:0:(service.c:1502:ptlrpc_at_check_timed()) Skipped 4 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x32/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: ffff8807fdaf0fd8 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2554e <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: e0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 45 31 e4 48 8d 86 d8 0f 00 00 48 89 7d 88 <48> 89 4d 98 4c 89 45 90 48 89 f3 49 89 d7 48 89 85 78 ff ff ff <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 8 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3f6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee <48> 8b bd 60 ff ff ff ff 95 58 ff ff ff 83 f8 03 0f 84 64 01 00 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Timed out RDMA with 10.10.32.107@o2ib2 (146): c: 8, oc: 0, rc: 8 <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Timed out RDMA with 10.10.32.135@o2ib2 (144): c: 8, oc: 0, rc: 8 <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Skipped 1 previous similar message <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Timed out RDMA with 10.10.32.102@o2ib2 (151): c: 8, oc: 0, rc: 8 <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Skipped 1 previous similar message <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Timed out RDMA with 10.10.32.230@o2ib2 (202): c: 8, oc: 0, rc: 8 <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Skipped 1 previous similar message <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Timed out RDMA with 10.10.32.72@o2ib2 (202): c: 8, oc: 0, rc: 8 <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Skipped 1 previous similar message <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Timed out tx: active_txs, 0 seconds <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Skipped 2 previous similar messages <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Timed out RDMA with 10.10.32.5@o2ib2 (202): c: 8, oc: 0, rc: 8 <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Skipped 2 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xcf/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: ffff8807fdaf0fd8 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> ffff8806db06bdcf 0000000000000004 ffff8807fdaf1080 ffff8806db06bda0 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 89 49 08 66 ff 02 66 66 90 48 83 c1 10 31 c0 49 89 4d 00 80 8b 3c 14 00 00 01 48 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 <4c> 8b 7d f8 c9 c3 0f 1f 00 66 3d 90 01 75 1d 31 d2 31 f6 bf 90 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Timed out tx: active_txs, 1 seconds <3>LNetError: 16707:0:(o2iblnd_cb.c:3153:kiblnd_check_txs_locked()) Skipped 13 previous similar messages <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Timed out RDMA with 10.10.32.133@o2ib2 (223): c: 8, oc: 0, rc: 8 <3>LNetError: 16707:0:(o2iblnd_cb.c:3216:kiblnd_check_conns()) Skipped 13 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3d5/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 <48> 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3b9/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff8806db06bdcf RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 48 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 62 <45> 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <3>LustreError: 17873:0:(ldlm_request.c:106:ldlm_expired_completion_wait()) ### lock timed out (enqueued at 1504701320, 750s ago); not entering recovery in server code, just going back to sleep ns: filter-f1-OST00bf_UUID lock: ffff880155448cc0/0x972469791f710474 lrc: 3/0,1 mode: --/PW res: [0xa2ec79a:0x0:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->18446744073709551615) flags: 0x40010080000000 nid: local remote: 0x0 expref: -99 pid: 17873 timeout: 0 lvb_type: 0 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3b9/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff8806db06bdcf RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 48 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 62 <45> 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x4c3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 83 f8 05 0f 84 67 03 00 00 83 f8 06 0f 84 a6 04 00 00 41 89 c1 4c 8d 45 cf 48 8b 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff ff ff 95 50 ff ff ff 85 c0 0f 84 d2 fe ff ff 48 8b 7d a8 48 85 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x28b/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2552b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 ff ff ff 48 89 c7 e8 c4 34 18 e0 41 8b 47 18 41 8b 57 14 83 c0 01 41 89 47 18 01 d0 41 83 fe 02 89 83 34 14 00 00 41 0f 94 c6 <4d> 85 ed 74 09 4d 85 e4 0f 85 d7 00 00 00 45 84 f6 0f 85 a9 00 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3b1/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: 000000000000004d RSI: ffff8806db06bdcf RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 40 18 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 48 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 <89> d1 48 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x16/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c <48> 83 39 3f 76 07 0f b7 56 32 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0x1b/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000286 <4>RAX: ffff8807fc043400 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000008 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06bd00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 ffff8806db06bdcf ffff8807fc043400 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 89 e5 48 81 ec a0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 6d e8 4c 89 75 f0 <4c> 89 7d f8 0f 1f 44 00 00 48 8b 05 d5 ce 73 ff 49 89 ff 48 89 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x4b1/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 e8 54 e4 6f ff e9 d2 fb ff ff 0f 1f 80 00 00 00 00 83 f8 05 0f 84 67 03 00 00 83 f8 06 0f 84 a6 04 00 00 41 89 c1 4c 8d 45 cf <48> 8b 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff ff ff ff 95 50 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3a3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f7 71 18 89 55 b8 48 8b 87 90 02 00 00 8b 40 18 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 <48> 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x4b1/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 e8 54 e4 6f ff e9 d2 fb ff ff 0f 1f 80 00 00 00 00 83 f8 05 0f 84 67 03 00 00 83 f8 06 0f 84 a6 04 00 00 41 89 c1 4c 8d 45 cf <48> 8b 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff ff ff ff 95 50 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0xf/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000282 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff03 ffffffffa0f2554e <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ba 16 00 00 00 48 89 04 24 31 c0 e8 7c 05 70 ff e9 61 fe ff ff 0f 1f 80 00 00 00 00 55 48 89 e5 48 81 ec e0 00 00 00 48 89 5d d8 <4c> 89 65 e0 4c 89 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x16/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c <48> 83 39 3f 76 07 0f b7 56 32 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0xf/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000282 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f25577 <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ba 16 00 00 00 48 89 04 24 31 c0 e8 7c 05 70 ff e9 61 fe ff ff 0f 1f 80 00 00 00 00 55 48 89 e5 48 81 ec e0 00 00 00 48 89 5d d8 <4c> 89 65 e0 4c 89 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3d5/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 <48> 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000286 <4>RAX: ffff8807fc043400 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000008 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06bd00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 ffff8806db06bdcf ffff8807fc043400 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: b2 02 00 01 00 00 00 e8 d4 38 70 ff e9 14 ff ff ff 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 89 e5 48 81 ec a0 00 00 00 <48> 89 5d d8 4c 89 65 e0 4c 89 6d e8 4c 89 75 f0 4c 89 7d f8 0f <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x23/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 3f 76 07 0f b7 56 32 c1 e2 10 <0f> b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x1/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a9c9 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 8b 90 d8 01 00 00 89 54 24 08 48 8b 40 40 ba 16 00 00 00 48 89 04 24 31 c0 e8 7c 05 70 ff e9 61 fe ff ff 0f 1f 80 00 00 00 00 55 <48> 89 e5 48 81 ec e0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 6d <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xbf/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000008 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 ffff8806db06bdcf ffff8807fc043400 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 8b 31 48 8b 41 08 48 89 46 08 48 89 30 48 89 09 48 89 49 08 66 ff 02 66 66 90 48 83 c1 10 31 c0 49 89 4d 00 80 8b 3c 14 00 00 01 <48> 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 4c 8b 7d f8 c9 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: ffff8807fdaf10a8 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000004 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000001 0000000000000004 ffff8806db06bdc0 ffff8806db06bda0 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 89 ff 48 89 f3 49 89 d4 49 89 cd 45 89 c6 48 85 c0 0f 85 95 00 00 00 f6 83 20 10 00 00 08 0f 84 1f 02 00 00 48 8d 83 a8 10 00 00 <48> 3b 83 a8 10 00 00 0f 84 5c 01 00 00 48 8d 93 d8 10 00 00 48 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x366/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd08 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffffffffa0f2a9c9 00000001aca54dc0 ffff8807fc043400 ffff88080000004d <4> ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 <4> ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 14 00 00 0f 82 8c 00 00 00 48 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 4c 8b 7d f8 c9 66 0f 1f 84 00 00 00 00 00 41 f6 85 18 0a 00 00 08 0f 84 1b <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x3e0/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2555a <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 03 00 00 00 e8 c2 a6 13 e0 4c 89 fa 66 ff 02 66 66 90 e9 cd fe ff ff 0f 1f 40 00 48 8b 4d 90 45 31 db c6 01 00 e9 6f ff ff ff 90 <45> 31 f6 e9 a3 fe ff ff 0f 1f 84 00 00 00 00 00 f6 05 dd b1 71 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xd4/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd08 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffffffffa0f2a903 00000001aca54dc0 ffff8807fc043400 ffff88080000004d <4> ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 <4> ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 02 66 66 90 48 83 c1 10 31 c0 49 89 4d 00 80 8b 3c 14 00 00 01 48 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 4c 8b 7d f8 c9 0f 1f 00 66 3d 90 01 75 1d 31 d2 31 f6 bf 90 01 00 00 e8 c4 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x55/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2552b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: d8 0f 00 00 48 89 7d 88 48 89 4d 98 4c 89 45 90 48 89 f3 49 89 d7 48 89 85 78 ff ff ff 4c 8b ae d0 0f 00 00 45 89 ce 49 8d 45 08 <4d> 85 ed 4c 0f 45 e0 41 83 f9 04 0f 84 7a 03 00 00 41 83 f9 07 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x0/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd08 EFLAGS: 00000282 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 ffff88080000004d <4> ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 <4> ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 10 48 83 39 3f 76 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 <55> 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x344/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff02 ffffffffa0f25577 <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f6 83 3c 14 00 00 02 0f 85 42 01 00 00 4d 85 e4 74 19 41 8b 84 24 0c 0a 00 00 05 00 04 00 00 39 83 34 14 00 00 0f 82 8c 00 00 00 <48> 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x16/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c <48> 83 39 3f 76 07 0f b7 56 32 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <6>usb 1-1.6: USB disconnect, device number 3 <6>usb 1-1.6.1: USB disconnect, device number 4 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3b3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff8806db06bdcf RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 48 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 <48> 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3f6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee <48> 8b bd 60 ff ff ff ff 95 58 ff ff ff 83 f8 03 0f 84 64 01 00 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <6>SysRq : HELP : loglevel(0-9) reBoot Crash terminate-all-tasks(E) memory-full-oom-kill(F) kill-all-tasks(I) thaw-filesystems(J) saK show-backtrace-all-active-cpus(L) show-memory-usage(M) nice-all-RT-tasks(N) powerOff show-registers(P) show-all-timers(Q) unRaw Sync show-task-states(T) Unmount show-blocked-tasks(W) dump-ftrace-buffer(Z) <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3a3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f7 71 18 89 55 b8 48 8b 87 90 02 00 00 8b 40 18 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 <48> 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3f6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee <48> 8b bd 60 ff ff ff ff 95 58 ff ff ff 83 f8 03 0f 84 64 01 00 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <6>SysRq : HELP : loglevel(0-9) reBoot Crash terminate-all-tasks(E) memory-full-oom-kill(F) kill-all-tasks(I) thaw-filesystems(J) saK show-backtrace-all-active-cpus(L) show-memory-usage(M) nice-all-RT-tasks(N) powerOff show-registers(P) show-all-timers(Q) unRaw Sync show-task-states(T) Unmount show-blocked-tasks(W) dump-ftrace-buffer(Z) <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x2cd/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff03 ffffffffa0f25577 <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f6 83 3c 14 00 00 04 0f 85 9c 00 00 00 4d 85 e4 74 19 41 8b 84 24 0c 0a 00 00 05 00 04 00 00 39 83 34 14 00 00 0f 82 03 01 00 00 <4d> 85 ed 74 54 4c 8d b3 d8 10 00 00 4c 89 f7 e8 1f 5b 62 e0 41 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xbf/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000000 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000001 ffff8807fc043400 ffff8806db06bdc0 ffff8806db06bda0 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 8b 31 48 8b 41 08 48 89 46 08 48 89 30 48 89 09 48 89 49 08 66 ff 02 66 66 90 48 83 c1 10 31 c0 49 89 4d 00 80 8b 3c 14 00 00 01 <48> 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 4c 8b 7d f8 c9 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x40/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: ffff8807fdaf0fd8 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f255ef <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xcf/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 45 31 e4 48 8d 86 d8 0f 00 00 48 89 7d 88 48 89 4d 98 4c 89 45 90 48 89 f3 49 89 d7 <48> 89 85 78 ff ff ff 4c 8b ae d0 0f 00 00 45 89 ce 49 8d 45 08 <4>Call Trace: <4> [] ? osd_scrub_next+0xcf/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x27/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd08 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 ffff88080000004d <4> ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 <4> ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 3f 76 07 0f b7 56 32 c1 e2 10 0f b7 c0 c9 <09> d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 89 16 48 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xcf/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000008 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> ffff8806db06bdcf ffff8807fc043000 000000000000004d 000000000000004d <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 89 49 08 66 ff 02 66 66 90 48 83 c1 10 31 c0 49 89 4d 00 80 8b 3c 14 00 00 01 48 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 <4c> 8b 7d f8 c9 c3 0f 1f 00 66 3d 90 01 75 1d 31 d2 31 f6 bf 90 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x55/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2552b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: d8 0f 00 00 48 89 7d 88 48 89 4d 98 4c 89 45 90 48 89 f3 49 89 d7 48 89 85 78 ff ff ff 4c 8b ae d0 0f 00 00 45 89 ce 49 8d 45 08 <4d> 85 ed 4c 0f 45 e0 41 83 f9 04 0f 84 7a 03 00 00 41 83 f9 07 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x498/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 ff 04 00 00 48 c7 05 63 4c 02 00 00 00 00 00 c7 05 51 4c 02 00 01 00 00 00 e8 54 e4 6f ff e9 d2 fb ff ff 0f 1f 80 00 00 00 00 <83> f8 05 0f 84 67 03 00 00 83 f8 06 0f 84 a6 04 00 00 41 89 c1 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x4/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000282 <4>RAX: ffff8807fc043000 RBX: ffff8806db06bd00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 8f 90 02 00 00 31 d2 0f b7 46 10 48 83 39 3f 76 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 <0f> 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x299/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff03 ffffffffa0f25577 <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 47 18 41 8b 57 14 83 c0 01 41 89 47 18 01 d0 41 83 fe 02 89 83 34 14 00 00 41 0f 94 c6 4d 85 ed 74 09 4d 85 e4 0f 85 d7 00 00 00 <45> 84 f6 0f 85 a9 00 00 00 f6 83 3c 14 00 00 04 0f 85 9c 00 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x40/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: ffff8807fdaf0fd8 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f25577 <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 45 31 e4 48 8d 86 d8 0f 00 00 48 89 7d 88 48 89 4d 98 4c 89 45 90 48 89 f3 49 89 d7 <48> 89 85 78 ff ff ff 4c 8b ae d0 0f 00 00 45 89 ce 49 8d 45 08 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: ffff8807fdaf10a8 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000000 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06bd00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 0000000000000001 ffff8807fc043400 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 89 ff 48 89 f3 49 89 d4 49 89 cd 45 89 c6 48 85 c0 0f 85 95 00 00 00 f6 83 20 10 00 00 08 0f 84 1f 02 00 00 48 8d 83 a8 10 00 00 <48> 3b 83 a8 10 00 00 0f 84 5c 01 00 00 48 8d 93 d8 10 00 00 48 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3d5/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 <48> 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3f6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee <48> 8b bd 60 ff ff ff ff 95 58 ff ff ff 83 f8 03 0f 84 64 01 00 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x16/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c <48> 83 39 3f 76 07 0f b7 56 32 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x344/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2554e <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f6 83 3c 14 00 00 02 0f 85 42 01 00 00 4d 85 e4 74 19 41 8b 84 24 0c 0a 00 00 05 00 04 00 00 39 83 34 14 00 00 0f 82 8c 00 00 00 <48> 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3d5/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 <48> 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 44 0f b6 45 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0xf/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000282 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2555a <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ba 16 00 00 00 48 89 04 24 31 c0 e8 7c 05 70 ff e9 61 fe ff ff 0f 1f 80 00 00 00 00 55 48 89 e5 48 81 ec e0 00 00 00 48 89 5d d8 <4c> 89 65 e0 4c 89 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e6/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 48 3b 51 18 77 35 <44> 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x344/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f25577 <4> 0000000000000010 0000000000000202 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f6 83 3c 14 00 00 02 0f 85 42 01 00 00 4d 85 e4 74 19 41 8b 84 24 0c 0a 00 00 05 00 04 00 00 39 83 34 14 00 00 0f 82 8c 00 00 00 <48> 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 <4>Call Trace: <4> [] ? osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x326/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2553b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x1b/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 48 39 83 a8 10 00 00 0f 84 df 05 00 00 80 a3 3c 14 00 00 fd 4c 89 f0 66 ff 00 66 66 90 f6 83 3c 14 00 00 02 0f 85 42 01 00 00 <4d> 85 e4 74 19 41 8b 84 24 0c 0a 00 00 05 00 04 00 00 39 83 34 <4>Call Trace: <4> [] ? osd_scrub_next+0x1b/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0xf/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000282 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2553b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x1b/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ba 16 00 00 00 48 89 04 24 31 c0 e8 7c 05 70 ff e9 61 fe ff ff 0f 1f 80 00 00 00 00 55 48 89 e5 48 81 ec e0 00 00 00 48 89 5d d8 <4c> 89 65 e0 4c 89 6d e8 4c 89 75 f0 4c 89 7d f8 0f 1f 44 00 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x1b/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3b3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff8806db06bdcf RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 48 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 <48> 3b 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x4/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000282 <4>RAX: ffff8807fc043000 RBX: ffff8806db06bd00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 8b 8f 90 02 00 00 31 d2 0f b7 46 10 48 83 39 3f 76 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 <0f> 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xbf/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000000 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> ffff8806db06bdcf 0000000000000000 ffff8807fc043000 0000000000000000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 8b 31 48 8b 41 08 48 89 46 08 48 89 30 48 89 09 48 89 49 08 66 ff 02 66 66 90 48 83 c1 10 31 c0 49 89 4d 00 80 8b 3c 14 00 00 01 <48> 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 4c 8b 7d f8 c9 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3e0/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 000000000000004d RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 48 18 73 62 45 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 20 ff ff ff 8d 14 10 48 8b 45 a0 48 8b 88 90 02 00 00 <48> 3b 51 18 77 35 44 0f b6 45 cf 48 8d 4d c0 48 8d 55 a0 4c 89 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x3e0/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2552b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 03 00 00 00 e8 c2 a6 13 e0 4c 89 fa 66 ff 02 66 66 90 e9 cd fe ff ff 0f 1f 40 00 48 8b 4d 90 45 31 db c6 01 00 e9 6f ff ff ff 90 <45> 31 f6 e9 a3 fe ff ff 0f 1f 84 00 00 00 00 00 f6 05 dd b1 71 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x326/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff03 ffffffffa0f2555a <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 48 39 83 a8 10 00 00 0f 84 df 05 00 00 80 a3 3c 14 00 00 fd 4c 89 f0 66 ff 00 66 66 90 f6 83 3c 14 00 00 02 0f 85 42 01 00 00 <4d> 85 e4 74 19 41 8b 84 24 0c 0a 00 00 05 00 04 00 00 39 83 34 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x27/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd08 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 ffff88080000004d <4> ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 <4> ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c 48 83 39 3f 76 07 0f b7 56 32 c1 e2 10 0f b7 c0 c9 <09> d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 89 16 48 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x4b1/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 e8 54 e4 6f ff e9 d2 fb ff ff 0f 1f 80 00 00 00 00 83 f8 05 0f 84 67 03 00 00 83 f8 06 0f 84 a6 04 00 00 41 89 c1 4c 8d 45 cf <48> 8b 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff ff ff ff 95 50 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3a3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f7 71 18 89 55 b8 48 8b 87 90 02 00 00 8b 40 18 0f af c6 83 c0 01 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 <48> 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x299/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2554e <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 47 18 41 8b 57 14 83 c0 01 41 89 47 18 01 d0 41 83 fe 02 89 83 34 14 00 00 41 0f 94 c6 4d 85 ed 74 09 4d 85 e4 0f 85 d7 00 00 00 <45> 84 f6 0f 85 a9 00 00 00 f6 83 3c 14 00 00 04 0f 85 9c 00 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd08 EFLAGS: 00000283 <4>RAX: ffff8807fc043400 RBX: ffff8806db06be00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffffffffa0f2a903 00000001aca54dc0 ffff8807fc043400 ffff88080000004d <4> ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 <4> ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: a3 b2 02 00 00 00 00 00 c7 05 91 b2 02 00 01 00 00 00 e8 d4 38 70 ff e9 14 ff ff ff 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 <55> 48 89 e5 48 81 ec a0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x1/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a9c9 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 8b 90 d8 01 00 00 89 54 24 08 48 8b 40 40 ba 16 00 00 00 48 89 04 24 31 c0 e8 7c 05 70 ff e9 61 fe ff ff 0f 1f 80 00 00 00 00 55 <48> 89 e5 48 81 ec e0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 6d <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0x57/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: ffff8807fdaf10a8 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000008 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 ffff8806db06bdcf ffff8807fc043400 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 89 ff 48 89 f3 49 89 d4 49 89 cd 45 89 c6 48 85 c0 0f 85 95 00 00 00 f6 83 20 10 00 00 08 0f 84 1f 02 00 00 48 8d 83 a8 10 00 00 <48> 3b 83 a8 10 00 00 0f 84 5c 01 00 00 48 8d 93 d8 10 00 00 48 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x3b9/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000283 <4>RAX: ffff8807fc043000 RBX: ffff8806db06be00 RCX: 000000000000004d <4>RDX: 000000000000004d RSI: ffff8806db06bdcf RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06be00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 89 45 b4 e8 0a f2 f4 ff 48 85 c0 48 89 45 a8 0f 84 35 08 00 00 48 8b 7d a0 8b 55 b8 48 8b 87 90 02 00 00 89 d1 48 3b 48 18 73 62 <45> 3b 37 76 5d 48 89 de 89 95 20 ff ff ff e8 b4 ff f6 ff 8b 95 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_inode_iteration+0x4c3/0xd70 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd10 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bd00 <4>R13: ffff8807fdaf0fd8 R14: ffff8807fdaf10b8 R15: ffff8807986e0000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> 00000001aca54dc0 ffff8807fc043400 ffff88080000004d ffff8806db06be10 <4> ffff8806db06bdd0 ffff8807fdaf1030 ffff8807986e0040 ffff8807986e0040 <4> ffffffffa0f28870 ffffffffa0f25520 ffff880c88e2d000 ffff880c0151e000 <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 83 f8 05 0f 84 67 03 00 00 83 f8 06 0f 84 a6 04 00 00 41 89 c1 4c 8d 45 cf 48 8b 4d c0 48 8d 55 a0 4c 89 ee 48 8b bd 60 ff ff ff 95 50 ff ff ff 85 c0 0f 84 d2 fe ff ff 48 8b 7d a8 48 85 ff <4>Call Trace: <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0x1b/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000286 <4>RAX: ffff8807fc043400 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000008 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06be00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 ffff8806db06bdcf ffff8807fc043400 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: ff 66 66 66 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 89 e5 48 81 ec a0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 6d e8 4c 89 75 f0 <4c> 89 7d f8 0f 1f 44 00 00 48 8b 05 d5 ce 73 ff 49 89 ff 48 89 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_next+0xbf/0x4c0 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc60 EFLAGS: 00000202 <4>RAX: 0000000000000004 RBX: ffff8806db06bd00 RCX: ffff8806db06bdc0 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: 0000000000000001 R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf1080 <4>R13: 0000000000000008 R14: ffff8806db06bdcf R15: 0000000000000004 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4> ffff8806db06bd00 ffff880c88e2d000 ffff8806db06bdcf ffff8807fdaf1080 <4> 0000000000000000 0000000000000004 ffff8806db06bdcf 0000000000000000 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 8b 31 48 8b 41 08 48 89 46 08 48 89 30 48 89 09 48 89 49 08 66 ff 02 66 66 90 48 83 c1 10 31 c0 49 89 4d 00 80 8b 3c 14 00 00 01 <48> 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 4c 8b 7d f8 c9 <4>Call Trace: <4> [] ? osd_inode_iteration+0x403/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Unexpectedly long timeout: desc ffff880a25626000 <4>Lustre: 17887:0:(niobuf.c:292:ptlrpc_abort_bulk()) Skipped 9 previous similar messages <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x361/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff02 ffffffffa0f2555a <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 00 39 83 34 14 00 00 0f 82 8c 00 00 00 48 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 <4c> 8b 7d f8 c9 c3 66 0f 1f 84 00 00 00 00 00 41 f6 85 18 0a 00 <4>Call Trace: <4> [] ? osd_scrub_next+0x3a/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] ldiskfs_itable_unused_count+0x16/0x30 [ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fc043000 <4>RDX: 0000000000000000 RSI: ffff88080fdd3000 RDI: ffff8807fc043400 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a8cc 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 07 0f b7 56 30 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 0f 1f 44 00 00 55 48 89 e5 0f 1f 44 00 00 48 8b 8f 90 02 00 00 31 d2 0f b7 46 1c <48> 83 39 3f 76 07 0f b7 56 32 c1 e2 10 0f b7 c0 c9 09 d0 c3 66 <4>Call Trace: <4> [] ? osd_inode_iteration+0x3cc/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x1/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bd00 EFLAGS: 00000293 <4>RAX: 0000000000000004 RBX: ffff8806db06be00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8807fdaf10b8 <4>R13: ffff8807986e0000 R14: ffff8807fdaf1090 R15: 0000000000000018 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8806db06be00 ffffffffa0f2a9c9 00000001aca54dc0 ffff8807fc043400 <4> ffff88080000004d ffff8806db06be10 ffff8806db06bdd0 ffff8807fdaf1030 <4> ffff8807986e0040 ffff8807986e0040 ffffffffa0f28870 ffffffffa0f25520 <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 8b 90 d8 01 00 00 89 54 24 08 48 8b 40 40 ba 16 00 00 00 48 89 04 24 31 c0 e8 7c 05 70 ff e9 61 fe ff ff 0f 1f 80 00 00 00 00 55 <48> 89 e5 48 81 ec e0 00 00 00 48 89 5d d8 4c 89 65 e0 4c 89 6d <4>Call Trace: <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 68s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x344/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000008 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8807fdaf0000 RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2554e <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: f6 83 3c 14 00 00 02 0f 85 42 01 00 00 4d 85 e4 74 19 41 8b 84 24 0c 0a 00 00 05 00 04 00 00 39 83 34 14 00 00 0f 82 8c 00 00 00 <48> 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 <4>Call Trace: <4> [] ? osd_scrub_next+0x2e/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <0>BUG: soft lockup - CPU#9 stuck for 67s! [OI_scrub:26550] <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4>CPU 9 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 26550, comm: OI_scrub Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] osd_scrub_exec+0x361/0x1c90 [osd_ldiskfs] <4>RSP: 0018:ffff8806db06bc20 EFLAGS: 00000246 <4>RAX: 0000000000000000 RBX: ffff8806db06bd00 RCX: ffff8807fdaf1080 <4>RDX: ffff8806db06bda0 RSI: ffff8806db06bdcf RDI: ffff880c88e2d000 <4>RBP: ffffffff8100bc0e R08: ffff8806db06bdcf R09: 0000000000000004 <4>R10: 0000000000000000 R11: 0000000000000000 R12: ffff8806db06bbe0 <4>R13: ffffffffa0f11552 R14: ffff8806db06bb90 R15: ffff880c88e2d000 <4>FS: 0000000000000000(0000) GS:ffff88089c420000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000417f55 CR3: 0000000001a8d000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process OI_scrub (pid: 26550, threadinfo ffff8806db068000, task ffff8807986e0040) <4>Stack: <4> ffff8807fdaf0000 ffff880c88e2d000 ffffffffffffff10 ffffffffa0f2552b <4> 0000000000000010 0000000000000286 ffff8806db06bc60 0000000000000018 <4> ffff8807fdaf1090 ffff8807986e0000 ffff8807fdaf10b8 ffff8807fdaf0fd8 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <4>Code: 00 00 39 83 34 14 00 00 0f 82 8c 00 00 00 48 8b 75 90 c6 06 01 45 31 db 44 89 d8 48 8b 5d d8 4c 8b 65 e0 4c 8b 6d e8 4c 8b 75 f0 <4c> 8b 7d f8 c9 c3 66 0f 1f 84 00 00 00 00 00 41 f6 85 18 0a 00 <4>Call Trace: <4> [] ? osd_scrub_next+0xb/0x4c0 [osd_ldiskfs] <4> [] ? osd_inode_iteration+0x4c9/0xd70 [osd_ldiskfs] <4> [] ? osd_scrub_exec+0x0/0x1c90 [osd_ldiskfs] <4> [] ? osd_scrub_next+0x0/0x4c0 [osd_ldiskfs] <4> [] ? osd_scrub_main+0x885/0xec0 [osd_ldiskfs] <4> [] ? schedule+0x3ee/0xb70 <4> [] ? osd_scrub_main+0x0/0xec0 [osd_ldiskfs] <4> [] ? kthread+0x9e/0xc0 <4> [] ? child_rip+0xa/0x20 <4> [] ? kthread+0x0/0xc0 <4> [] ? child_rip+0x0/0x20 <6>SysRq : Trigger a crash <1>BUG: unable to handle kernel NULL pointer dereference at (null) <1>IP: [] sysrq_handle_crash+0x16/0x20 <4>PGD 105df72067 PUD 105d8c9067 PMD 0 <4>Oops: 0002 [#1] SMP <4>last sysfs file: /sys/devices/pci0000:00/0000:00:02.0/0000:03:00.0/host7/target7:0:0/7:0:0:115/timeout <4>CPU 8 <4>Modules linked in: osp(U) ofd(U) lfsck(U) ost(U) mgc(U) osd_ldiskfs(U) ldiskfs(U) mbcache jbd2 lquota(U) lustre(U) lov(U) mdc(U) fid(U) lmv(U) fld(U) ko2iblnd(U) ptlrpc(U) obdclass(U) lnet(U) sha512_generic crc32c_intel libcfs(U) ipmi_si dm_round_robin sd_mod crc_t10dif sg ib_srp scsi_transport_srp scsi_tgt mpt3sas mpt2sas scsi_transport_sas raid_class mptctl mptbase dell_rbu autofs4 ipmi_devintf 8021q garp stp llc nf_conntrack_netbios_ns nf_conntrack_broadcast ipt_REJECT nf_conntrack_ipv4 nf_defrag_ipv4 xt_state nf_conntrack iptable_filter ip_tables ib_ipoib rdma_ucm ib_ucm ib_uverbs ib_umad rdma_cm ib_cm iw_cm mlx4_ib ib_sa ib_mad ib_core ib_addr dm_mirror dm_region_hash dm_log dm_multipath dm_mod ipmi_msghandler microcode iTCO_wdt iTCO_vendor_support wmi acpi_pad dcdbas joydev shpchp sb_edac edac_core i2c_i801 ahci lpc_ich mfd_core ipv6 nfs lockd fscache auth_rpcgss nfs_acl sunrpc mlx4_en mlx4_core igb dca i2c_algo_bit i2c_core ptp pps_core [last unloaded: ipmi_si] <4> <4>Pid: 0, comm: swapper Tainted: G --L------------ 2.6.32-696.3.2.el6.atlas.x86_64 #1 Dell Inc. PowerEdge C6220 II/09N44V <4>RIP: 0010:[] [] sysrq_handle_crash+0x16/0x20 <4>RSP: 0000:ffff88089c403db8 EFLAGS: 00010092 <4>RAX: 0000000000000010 RBX: 0000000000000063 RCX: 000000000000fa77 <4>RDX: 0000000000000000 RSI: ffff88106b23b800 RDI: 0000000000000063 <4>RBP: ffff88089c403db8 R08: 0000000000000000 R09: ffff8800000b9e00 <4>R10: 0000000000000000 R11: 0000000000000198 R12: ffff88106b23b800 <4>R13: ffffffff81b13680 R14: 0000000000000082 R15: 0000000000000007 <4>FS: 0000000000000000(0000) GS:ffff88089c400000(0000) knlGS:0000000000000000 <4>CS: 0010 DS: 0018 ES: 0018 CR0: 000000008005003b <4>CR2: 0000000000000000 CR3: 000000106ab6e000 CR4: 00000000001407e0 <4>DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 <4>DR3: 0000000000000000 DR6: 00000000ffff0ff0 DR7: 0000000000000400 <4>Process swapper (pid: 0, threadinfo ffff8808742b0000, task ffff8808742a4040) <4>Stack: <4> ffff88089c403e08 ffffffff8135efc2 ffff88089c416f00 ffff880800000001 <4> 0000000000000008 ffffffff82022778 0000000000000100 0000000000000061 <4> 0000000000000000 0000000000000296 ffff88089c403e18 ffffffff8135f0ab <4>Call Trace: <4> <4> [] __handle_sysrq+0x132/0x1a0 <4> [] handle_sysrq+0x2b/0x30 <4> [] serial8250_handle_port+0x218/0x320 <4> [] serial8250_interrupt+0x8b/0x120 <4> [] handle_IRQ_event+0x60/0x170 <4> [] handle_edge_irq+0xde/0x180 <4> [] handle_irq+0x49/0xa0 <4> [] do_IRQ+0x6c/0xf0 <4> [] ret_from_intr+0x0/0x11 <4> <4> [] ? intel_idle+0xfe/0x1b0 <4> [] ? intel_idle+0xe1/0x1b0 <4> [] ? sched_clock+0x9/0x10 <4> [] ? sched_clock_cpu+0xcd/0x110 <4> [] cpuidle_idle_call+0x7a/0xe0 <4> [] cpu_idle+0xb6/0x110 <4> [] start_secondary+0x2c0/0x316 <4>Code: d0 88 81 e3 fb ff 81 c9 c3 66 66 66 2e 0f 1f 84 00 00 00 00 00 55 48 89 e5 0f 1f 44 00 00 c7 05 6d d5 74 00 01 00 00 00 0f ae f8 04 25 00 00 00 00 01 c9 c3 55 48 89 e5 0f 1f 44 00 00 8d 47 <1>RIP [] sysrq_handle_crash+0x16/0x20 <4> RSP <4>CR2: 0000000000000000