May-26 08:51:27 [root@lascaux1234 127.0.0.1-2014-05-20-09:47:30] # EDITOR=emacs /ptmp/crash/root/bin/crash $(pwd)/vmlinux $(pwd)/vmcore crash 6.1.0-1.el6 Copyright (C) 2002-2012 Red Hat, Inc. Copyright (C) 2004, 2005, 2006, 2010 IBM Corporation Copyright (C) 1999-2006 Hewlett-Packard Co Copyright (C) 2005, 2006, 2011, 2012 Fujitsu Limited Copyright (C) 2006, 2007 VA Linux Systems Japan K.K. Copyright (C) 2005, 2011 NEC Corporation Copyright (C) 1999, 2002, 2007 Silicon Graphics, Inc. Copyright (C) 1999, 2000, 2001, 2002 Mission Critical Linux, Inc. This program is free software, covered by the GNU General Public License, and you are welcome to change it and/or distribute copies of it under certain conditions. Enter "help copying" to see the conditions. This program has absolutely no warranty. Enter "help warranty" for details. GNU gdb (GDB) 7.3.1 Copyright (C) 2011 Free Software Foundation, Inc. License GPLv3+: GNU GPL version 3 or later This is free software: you are free to change and redistribute it. There is NO WARRANTY, to the extent permitted by law. Type "show copying" and "show warranty" for details. This GDB was configured as "x86_64-unknown-linux-gnu"... KERNEL: /ptmp/crash/lascaux110/127.0.0.1-2014-05-20-09:47:30/vmlinux DUMPFILE: /ptmp/crash/lascaux110/127.0.0.1-2014-05-20-09:47:30/vmcore [PARTIAL DUMP] CPUS: 32 DATE: Tue May 20 09:47:19 2014 UPTIME: 5 days, 22:45:15 LOAD AVERAGE: 0.49, 1.04, 0.79 TASKS: 1863 NODENAME: lascaux110 RELEASE: 2.6.32-431.5.1.el6.Bull.46.x86_64 VERSION: #1 SMP Sat Mar 1 01:42:26 CET 2014 MACHINE: x86_64 (2266 Mhz) MEMORY: 64 GB PANIC: "Kernel panic - not syncing: LBUG" PID: 23161 COMMAND: "mdt02_019" TASK: ffff880927d51500 [THREAD_INFO: ffff880aa2c06000] CPU: 22 STATE: TASK_RUNNING (PANIC) crash> log ..... ------------[ cut here ]------------ WARNING: at lib/list_debug.c:26 __list_add+0x6d/0xa0() (Tainted: G W --------------- ) Hardware name: bullx super-node list_add corruption. next->prev should be prev (ffff880bcf16e250), but was (null). (next=ffff88095658d280). LustreError: 23161:0:(ldlm_inodebits.c:108:ldlm_inodebits_compat_queue()) ASSERTION( lock->l_sl_mode.prev != ((void *)0) ) failed: LustreError: 23161:0:(ldlm_inodebits.c:108:ldlm_inodebits_compat_queue()) LBUG Pid: 23161, comm: mdt02_019 Call Trace: [] libcfs_debug_dumpstack+0x55/0x80 [libcfs] [] lbug_with_loc+0x47/0xb0 [libcfs] [] ldlm_inodebits_compat_queue+0x339/0x340 [ptlrpc] [] ? cfs_hash_bd_lookup_intent+0xdd/0x130 [libcfs] [] ldlm_process_inodebits_lock+0x182/0x3e0 [ptlrpc] [] ? ldlm_resource_get+0xcd/0x820 [ptlrpc] [] ldlm_lock_enqueue+0x405/0x8d0 [ptlrpc] [] ldlm_cli_enqueue_local+0x18b/0x5d0 [ptlrpc] [] ? ldlm_completion_ast+0x0/0x960 [ptlrpc] [] ? mdt_blocking_ast+0x0/0x2a0 [mdt] [] mdt_object_lock0+0x339/0xaf0 [mdt] [] ? mdt_blocking_ast+0x0/0x2a0 [mdt] [] ? ldlm_completion_ast+0x0/0x960 [ptlrpc] [] ? osd_xattr_get+0x21b/0x2d0 [osd_ldiskfs] [] mdt_object_lock_try+0x5a/0xb0 [mdt] [] ? mdd_xattr_get+0xac/0x280 [mdd] [] mdt_object_open_lock+0x2d0/0x510 [mdt] [] ? mdt_attr_get_complex+0x4ec/0x770 [mdt] [] mdt_reint_open+0x14aa/0x21c0 [mdt] [] ? upcall_cache_get_entry+0x28e/0x860 [libcfs] [] ? lustre_msg_add_version+0x6c/0xc0 [ptlrpc] [] mdt_reint_rec+0x41/0xe0 [mdt] [] mdt_reint_internal+0x4c3/0x780 [mdt] [] mdt_intent_reint+0x1f0/0x530 [mdt] [] mdt_intent_policy+0x39e/0x720 [mdt] [] ldlm_lock_enqueue+0x361/0x8d0 [ptlrpc] [] ldlm_handle_enqueue0+0x4ef/0x10b0 [ptlrpc] [] mdt_enqueue+0x46/0xe0 [mdt] [] mdt_handle_common+0x647/0x16d0 [mdt] [] ? lustre_msg_get_transno+0x8c/0x100 [ptlrpc] [] mds_regular_handle+0x15/0x20 [mdt] [] ptlrpc_server_handle_request+0x398/0xc60 [ptlrpc] [] ? cfs_timer_arm+0xe/0x10 [libcfs] [] ? lc_watchdog_touch+0x6f/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x290 [ptlrpc] [] ? __wake_up+0x53/0x70 [] ptlrpc_main+0xace/0x1700 [ptlrpc] [] ? ptlrpc_main+0x0/0x1700 [ptlrpc] [] child_rip+0xa/0x20 [] ? ptlrpc_main+0x0/0x1700 [ptlrpc] [] ? ptlrpc_main+0x0/0x1700 [ptlrpc] [] ? child_rip+0x0/0x20 Kernel panic - not syncing: LBUG Pid: 23161, comm: mdt02_019 Tainted: G W --------------- 2.6.32-431.5.1.el6.Bull.46.x86_64 #1 Call Trace: [] ? panic+0xa7/0x16f [] ? lbug_with_loc+0x9b/0xb0 [libcfs] [] ? ldlm_inodebits_compat_queue+0x339/0x340 [ptlrpc] [] ? cfs_hash_bd_lookup_intent+0xdd/0x130 [libcfs] [] ? ldlm_process_inodebits_lock+0x182/0x3e0 [ptlrpc] [] ? ldlm_resource_get+0xcd/0x820 [ptlrpc] [] ? ldlm_lock_enqueue+0x405/0x8d0 [ptlrpc] [] ? ldlm_cli_enqueue_local+0x18b/0x5d0 [ptlrpc] [] ? ldlm_completion_ast+0x0/0x960 [ptlrpc] [] ? mdt_blocking_ast+0x0/0x2a0 [mdt] [] ? mdt_object_lock0+0x339/0xaf0 [mdt] [] ? mdt_blocking_ast+0x0/0x2a0 [mdt] [] ? ldlm_completion_ast+0x0/0x960 [ptlrpc] [] ? osd_xattr_get+0x21b/0x2d0 [osd_ldiskfs] [] ? mdt_object_lock_try+0x5a/0xb0 [mdt] [] ? mdd_xattr_get+0xac/0x280 [mdd] [] ? mdt_object_open_lock+0x2d0/0x510 [mdt] [] ? mdt_attr_get_complex+0x4ec/0x770 [mdt] [] ? mdt_reint_open+0x14aa/0x21c0 [mdt] [] ? upcall_cache_get_entry+0x28e/0x860 [libcfs] [] ? lustre_msg_add_version+0x6c/0xc0 [ptlrpc] [] ? mdt_reint_rec+0x41/0xe0 [mdt] [] ? mdt_reint_internal+0x4c3/0x780 [mdt] [] ? mdt_intent_reint+0x1f0/0x530 [mdt] [] ? mdt_intent_policy+0x39e/0x720 [mdt] [] ? ldlm_lock_enqueue+0x361/0x8d0 [ptlrpc] [] ? ldlm_handle_enqueue0+0x4ef/0x10b0 [ptlrpc] [] ? mdt_enqueue+0x46/0xe0 [mdt] [] ? mdt_handle_common+0x647/0x16d0 [mdt] [] ? lustre_msg_get_transno+0x8c/0x100 [ptlrpc] [] ? mds_regular_handle+0x15/0x20 [mdt] [] ? ptlrpc_server_handle_request+0x398/0xc60 [ptlrpc] [] ? cfs_timer_arm+0xe/0x10 [libcfs] [] ? lc_watchdog_touch+0x6f/0x170 [libcfs] [] ? ptlrpc_wait_event+0xa9/0x290 [ptlrpc] [] ? __wake_up+0x53/0x70 [] ? ptlrpc_main+0xace/0x1700 [ptlrpc] [] ? ptlrpc_main+0x0/0x1700 [ptlrpc] [] ? child_rip+0xa/0x20 [] ? ptlrpc_main+0x0/0x1700 [ptlrpc] [] ? ptlrpc_main+0x0/0x1700 [ptlrpc] [] ? child_rip+0x0/0x20 crash> bt PID: 23161 TASK: ffff880927d51500 CPU: 22 COMMAND: "mdt02_019" #0 [ffff880aa2c07558] machine_kexec at ffffffff8103915b #1 [ffff880aa2c075b8] crash_kexec at ffffffff810c5e42 #2 [ffff880aa2c07688] panic at ffffffff815283aa #3 [ffff880aa2c07708] lbug_with_loc at ffffffffa049beeb [libcfs] #4 [ffff880aa2c07728] ldlm_inodebits_compat_queue at ffffffffa0971749 [ptlrpc] #5 [ffff880aa2c07788] ldlm_process_inodebits_lock at ffffffffa09718d2 [ptlrpc] #6 [ffff880aa2c07808] ldlm_lock_enqueue at ffffffffa09438d5 [ptlrpc] #7 [ffff880aa2c07868] ldlm_cli_enqueue_local at ffffffffa09626db [ptlrpc] #8 [ffff880aa2c078f8] mdt_object_lock0 at ffffffffa063de89 [mdt] #9 [ffff880aa2c079a8] mdt_object_lock_try at ffffffffa063e69a [mdt] #10 [ffff880aa2c07a08] mdt_object_open_lock at ffffffffa0662d60 [mdt] #11 [ffff880aa2c07a88] mdt_reint_open at ffffffffa06699aa [mdt] #12 [ffff880aa2c07b78] mdt_reint_rec at ffffffffa0653f41 [mdt] #13 [ffff880aa2c07b98] mdt_reint_internal at ffffffffa0638c83 [mdt] #14 [ffff880aa2c07bd8] mdt_intent_reint at ffffffffa0639210 [mdt] #15 [ffff880aa2c07c28] mdt_intent_policy at ffffffffa06370be [mdt] #16 [ffff880aa2c07c68] ldlm_lock_enqueue at ffffffffa0943831 [ptlrpc] #17 [ffff880aa2c07cc8] ldlm_handle_enqueue0 at ffffffffa096a1df [ptlrpc] #18 [ffff880aa2c07d38] mdt_enqueue at ffffffffa0637546 [mdt] #19 [ffff880aa2c07d58] mdt_handle_common at ffffffffa063bce7 [mdt] #20 [ffff880aa2c07da8] mds_regular_handle at ffffffffa0677cc5 [mdt] #21 [ffff880aa2c07db8] ptlrpc_server_handle_request at ffffffffa099c428 [ptlrpc] #22 [ffff880aa2c07eb8] ptlrpc_main at ffffffffa099d7be [ptlrpc] #23 [ffff880aa2c07f48] kernel_thread at ffffffff8100c20a crash> crash> kmem -s CACHE NAME OBJSIZE ALLOCATED TOTAL SLABS SSIZE ffff880eda4c1a00 osp_obj 216 7402 8064 448 4k kmem: lod_obj: full list: slab: ffff880aec35c000 bad next pointer: 0 kmem: lod_obj: full list: slab: ffff880aec35c000 bad prev pointer: 0 kmem: lod_obj: full list: slab: ffff880aec35c000 bad inuse counter: 0 kmem: lod_obj: full list: slab: ffff880aec35c000 bad s_mem pointer: 0 ffff880f076f19c0 lod_obj 120 37883 38144 1192 4k kmem: mdt_obj: full list: slab: ffff880955f68000 bad next pointer: 0 kmem: mdt_obj: full list: slab: ffff880955f68000 bad prev pointer: 0 kmem: mdt_obj: full list: slab: ffff880955f68000 bad inuse counter: 0 kmem: mdt_obj: full list: slab: ffff880955f68000 bad s_mem pointer: 0 ffff880f49491980 mdt_obj 248 29851 29968 1873 4k ffff880f22b01940 fsfilt_ldiskfs_fcb 56 0 0 0 4k ffff880f5de71900 dynlock_cache 128 0 450 15 4k kmem: ldiskfs_inode_cache: full list: slab: ffff88095658d280 bad next pointer: 0 kmem: ldiskfs_inode_cache: full list: slab: ffff88095658d280 bad prev pointer: 0 kmem: ldiskfs_inode_cache: full list: slab: ffff88095658d280 bad inuse counter: 0 kmem: ldiskfs_inode_cache: full list: slab: ffff88095658d280 bad s_mem pointer: 0 ffff880f311718c0 ldiskfs_inode_cache 1048 29212 29226 9742 4k ffff880f5de81880 ldiskfs_xattr 88 0 0 0 4k ffff880f31041840 ldiskfs_free_data 64 1 59 1 4k ffff880f311c1800 ldiskfs_alloc_context 136 0 224 8 4k ffff880f22a517c0 ldiskfs_prealloc_space 112 34 136 4 4k ffff880f082f1780 ldiskfs_system_zone 40 0 0 0 4k kmem: mdd_obj: full list: slab: ffff880955f6d000 bad next pointer: 0 kmem: mdd_obj: full list: slab: ffff880955f6d000 bad prev pointer: 0 kmem: mdd_obj: full list: slab: ffff880955f6d000 bad inuse counter: 0 kmem: mdd_obj: full list: slab: ffff880955f6d000 bad s_mem pointer: 0 ffff880f313f1740 mdd_obj 96 29889 30280 757 4k ffff880f08e31700 ccc_req_kmem 40 0 0 0 4k ffff880f056c16c0 ccc_session_kmem 184 1 21 1 4k ffff880edce51680 ccc_thread_kmem 352 163 187 17 4k ffff880ed94e1640 ccc_object_kmem 264 0 0 0 4k ffff880edce21600 ccc_lock_kmem 40 0 0 0 4k ffff880f1cbe15c0 vvp_session_kmem 104 1 37 1 4k ffff880f067d1580 vvp_thread_kmem 488 163 176 22 4k ffff881075de1540 ll_rmtperm_hash_cache 256 0 0 0 4k ffff880f06cd1500 ll_remote_perm_cache 40 0 0 0 4k ffff880f076b14c0 ll_file_data 192 0 0 0 4k ffff880f054b1480 lustre_inode_cache 1216 0 0 0 4k ffff880ed94d1440 lov_oinfo 128 0 0 0 4k ffff880f07111400 lov_lock_link_kmem 32 0 0 0 4k ffff880f060813c0 lovsub_req_kmem 40 0 0 0 4k ffff880f08f01380 lovsub_object_kmem 240 0 0 0 4k ffff880f0a9c1340 lovsub_lock_kmem 64 0 0 0 4k ffff880f0a681300 lov_req_kmem 40 0 0 0 4k ffff880ed9ea12c0 lov_session_kmem 400 1 10 1 4k ffff880f07131280 lov_thread_kmem 288 163 208 16 4k ffff880f0f6d1240 lov_object_kmem 240 0 0 0 4k ffff880f062c1200 lov_lock_kmem 104 0 0 0 4k ffff880f057411c0 osc_quota_kmem 24 0 0 0 4k ffff880f06701180 osc_extent_kmem 168 0 0 0 4k ffff88105cac1140 osc_req_kmem 40 0 0 0 4k ffff880f06481100 osc_session_kmem 424 1 9 1 4k ffff880f077c10c0 osc_thread_kmem 984 163 168 42 4k ffff880f063f1080 osc_object_kmem 288 0 0 0 4k ffff880f08581040 osc_lock_kmem 192 0 0 0 4k ffff880849b20a00 upd_kmem 96 0 0 0 4k ffff8806efcc09c0 lqe_kmem 192 4 20 1 4k ffff880a83ac06c0 interval_node 128 0 0 0 4k kmem: ldlm_locks: full list: slab: ffff880955e2ca40 bad next pointer: 0 kmem: ldlm_locks: full list: slab: ffff880955e2ca40 bad prev pointer: 0 kmem: ldlm_locks: full list: slab: ffff880955e2ca40 bad inuse counter: 0 kmem: ldlm_locks: full list: slab: ffff880955e2ca40 bad s_mem pointer: 0 ffff880a77e20680 ldlm_locks 576 45435 46116 6588 4k kmem: ldlm_resources: full list: slab: ffff880955e2d040 bad next pointer: 0 kmem: ldlm_resources: full list: slab: ffff880955e2d040 bad prev pointer: 0 kmem: ldlm_resources: full list: slab: ffff880955e2d040 bad inuse counter: 0 kmem: ldlm_resources: full list: slab: ffff880955e2d040 bad s_mem pointer: 0 ffff880a6a7e0640 ldlm_resources 320 27296 27468 2289 4k ffff8806efd40980 cl_lock_kmem 224 0 0 0 4k ffff8807374d0940 cl_env_kmem 168 1 23 1 4k ffff8807324c0900 capa_cache 184 0 0 0 4k ffff8807363008c0 ll_import_cache 1408 0 0 0 8k ffff880744dc0880 ll_obdo_cache 208 0 0 0 4k ffff8807327b0840 ll_obd_dev_cache 6648 231 231 231 8k ffff88046f2c1540 nfs_direct_cache 200 0 0 0 4k ffff88046f2b1500 nfs_commit_data 704 0 0 0 8k ffff88046f2a14c0 nfs_write_data 960 36 36 9 4k ffff88046f291480 nfs_read_data 896 0 0 0 4k ffff88046f281440 nfs_inode_cache 1048 38 42 14 4k ffff88046f261400 nfs_page 128 0 0 0 4k ffff88046f1313c0 fscache_cookie_jar 80 4 48 1 4k ffff88046f111380 rpc_buffers 2048 8 8 4 4k ffff88046f101340 rpc_tasks 256 8 15 1 4k ffff88046f071300 rpc_inode_cache 832 22 24 6 4k ffff881078720e00 fib6_nodes 64 19 118 2 4k ffff881078710dc0 ip6_dst_cache 384 17 30 3 4k ffff881078700d80 ndisc_cache 256 0 0 0 4k ffff881076cf0d40 ip6_mrt_cache 128 0 0 0 4k ffff881076ce0d00 RAWv6 1024 35 36 9 4k ffff8810786f0cc0 UDPLITEv6 1024 0 0 0 4k ffff8810786e0c80 UDPv6 1024 7 16 4 4k ffff881076c10c40 tw_sock_TCPv6 320 0 0 0 4k ffff881076c00c00 request_sock_TCPv6 192 0 0 0 4k ffff8810786c0bc0 TCPv6 1920 4 6 3 4k ffff880877500800 ib_mad 448 2048 2480 310 4k ffff8808774f07c0 dm_mpath_io 16 2048 3030 15 4k ffff8804788112c0 ioat2 128 4096 4110 137 4k ffff8810790e0b80 scsi_tgt_cmd 80 0 0 0 4k ffff880477aa1280 ext4_inode_cache 1016 18985 19212 4803 4k ffff880477a91240 ext4_xattr 88 0 0 0 4k ffff880477a81200 ext4_free_block_extents 56 0 0 0 4k ffff880477a711c0 ext4_alloc_context 136 0 28 1 4k ffff880477a61180 ext4_prealloc_space 104 44 185 5 4k ffff880477a51140 ext4_system_zone 40 0 0 0 4k ffff880477971100 jbd2_journal_handle 24 0 1008 7 4k ffff8804779610c0 jbd2_journal_head 112 1126 4386 129 4k ffff880477951080 jbd2_revoke_table 16 8 606 3 4k ffff880477941040 jbd2_revoke_record 32 2 112 1 4k ffff8810784e0b40 sd_ext_cdb 32 2 112 1 4k ffff880478321000 scsi_sense_cache 128 14 420 14 4k ffff880478310fc0 scsi_cmd_cache 256 14 330 22 4k ffff880878a50780 dm_raid1_read_record 1064 0 0 0 8k ffff881078ac0b00 kcopyd_job 3240 0 0 0 8k ffff881078ab0ac0 io 64 0 0 0 4k ffff881078aa0a80 dm_uevent 2608 0 0 0 8k ffff881078a90a40 dm_rq_clone_bio_info 16 2048 3838 19 4k ffff881078a80a00 dm_rq_target_io 392 2048 2280 228 4k ffff881078a709c0 dm_target_io 24 0 0 0 4k ffff881078a60980 dm_io 40 0 0 0 4k ffff880c78930500 flow_cache 104 0 0 0 4k ffff8804787e0f80 uhci_urb_priv 56 0 0 0 4k ffff880878dd0740 cfq_io_context 136 21 140 5 4k ffff881078df0940 cfq_queue 240 4 48 3 4k ffff881078de0900 bsg_cmd 312 0 0 0 4k ffff880878da0700 mqueue_inode_cache 896 1 4 1 4k ffff880c78d004c0 isofs_inode_cache 640 0 0 0 4k ffff880478a30f40 hugetlbfs_inode_cache 608 1 6 1 4k ffff881078db08c0 dquot 256 0 0 0 4k ffff8804789a0f00 kioctx 384 0 0 0 4k ffff880c78cd0480 kiocb 256 0 0 0 4k ffff880478990ec0 inotify_event_private_data 32 0 112 1 4k ffff880c78cc0440 inotify_inode_mark_entry 112 138 374 11 4k ffff880478960e80 dnotify_mark_entry 112 6 34 1 4k ffff880c78cb0400 dnotify_struct 32 6 112 1 4k ffff880478950e40 dio 640 0 0 0 4k ffff880478940e00 fasync_cache 24 0 0 0 4k ffff880878d706c0 khugepaged_mm_slot 40 0 0 0 4k ffff880478930dc0 ksm_mm_slot 48 0 0 0 4k ffff880c78ca03c0 ksm_stable_node 48 0 0 0 4k ffff880478920d80 ksm_rmap_item 64 0 0 0 4k ffff880878d10680 utrace_engine 56 0 0 0 4k ffff881078d60880 utrace 64 0 0 0 4k ffff880878d00640 pid_namespace 2168 0 0 0 8k ffff881078d50840 posix_timers_cache 176 0 0 0 4k ffff880878cf0600 uid_cache 128 8 120 4 4k ffff881079620800 UNIX 768 143 225 45 4k ffff8808795a05c0 ip_mrt_cache 128 0 0 0 4k ffff880479220d40 UDP-Lite 832 0 0 0 8k ffff880c79ac0380 tcp_bind_bucket 64 12 177 3 4k ffff880c79ab0340 inet_peer_cache 64 1 59 1 4k ffff880c79aa0300 secpath_cache 64 0 0 0 4k ffff881079ba07c0 xfrm_dst_cache 448 0 0 0 4k ffff880c79a702c0 ip_fib_alias 32 0 0 0 4k ffff8804797c0d00 ip_fib_hash 72 24 106 2 4k ffff880c79970280 ip_dst_cache 384 37 210 21 4k ffff881079a60780 arp_cache 256 1082 2880 192 4k ffff8804796a0cc0 PING 832 0 0 0 8k ffff880c79960240 RAW 832 33 45 5 8k ffff880479690c80 UDP 832 16 45 5 8k ffff880c79950200 tw_sock_TCP 256 0 0 0 4k ffff881079a30740 request_sock_TCP 128 0 0 0 4k ffff880479680c40 TCP 1728 12 28 7 8k ffff8808799d0580 eventpoll_pwq 72 69 265 5 4k ffff881079a00700 eventpoll_epi 128 69 180 6 4k ffff8810799d06c0 sgpool-256 8192 2 4 4 8k ffff8810799c0680 sgpool-128 4096 2 3 3 4k ffff8810799b0640 sgpool-64 2048 2 14 7 4k ffff8810799a0600 sgpool-32 1024 2 24 6 4k ffff8810799905c0 sgpool-16 512 2 16 2 4k ffff881079960580 sgpool-8 256 5 345 23 4k ffff880879960540 scsi_data_buffer 24 0 0 0 4k ffff8804798e0c00 blkdev_integrity 112 0 0 0 4k ffff880879d20500 blkdev_queue 2864 62 62 31 8k ffff881079d30540 blkdev_requests 352 152 462 42 4k ffff880479850bc0 blkdev_ioc 80 9 192 4 4k ffff880c79c501c0 fsnotify_event_holder 24 0 0 0 4k ffff880479840b80 fsnotify_event 104 0 37 1 4k ffff880c79c40180 bio-0 192 2050 3820 191 4k ffff881079ce0500 biovec-256 4096 2050 2050 2050 4k ffff880879cf04c0 biovec-128 2048 0 0 0 4k ffff881079cd04c0 biovec-64 1024 0 0 0 4k ffff880879ce0480 biovec-16 256 0 0 0 4k ffff881079cc0480 bip-256 4224 2 2 2 8k ffff880879cd0440 bip-128 2176 0 0 0 8k ffff881079cb0440 bip-64 1152 0 0 0 8k ffff880879cc0400 bip-16 384 0 0 0 4k ffff881079ca0400 bip-4 192 0 0 0 4k ffff880879cb03c0 bip-1 128 0 0 0 4k ffff881079c503c0 sock_inode_cache 704 260 350 70 4k ffff880879c60380 skbuff_fclone_cache 512 0 14 2 4k ffff881079c40380 skbuff_head_cache 256 1288 1935 129 4k ffff880c7a7c0140 file_lock_cache 176 9 88 4 4k ffff881079c00340 net_namespace 2432 0 0 0 8k ffff88107a7b0300 shmem_inode_cache 784 1587 1685 337 4k ffff880c7ace0100 Acpi-Operand 72 166625 169388 3196 4k ffff88047a780b40 Acpi-ParseExt 72 0 0 0 4k ffff880c7a4f00c0 Acpi-Parse 48 0 0 0 4k ffff88047a770b00 Acpi-State 80 0 0 0 4k ffff880c7a500080 Acpi-Namespace 40 21512 21620 235 4k ffff88107a5202c0 task_delay_info 112 1865 2448 72 4k ffff88087a530340 taskstats 328 1 12 1 4k kmem: proc_inode_cache: full list: slab: ffff88095642e000 bad next pointer: 0 kmem: proc_inode_cache: full list: slab: ffff88095642e000 bad prev pointer: 0 kmem: proc_inode_cache: full list: slab: ffff88095642e000 bad inuse counter: 0 kmem: proc_inode_cache: full list: slab: ffff88095642e000 bad s_mem pointer: 0 ffff880c7a510040 proc_inode_cache 656 1979 2250 375 4k ffff88047a760ac0 sigqueue 160 1 48 2 4k ffff88087a510300 bdev_cache 832 61 96 24 4k ffff88047a730a80 sysfs_dir_cache 144 30595 30780 1140 4k ffff88087a4f02c0 mnt_cache 256 32 105 7 4k ffff88107a4e0280 filp 192 1141 3860 193 4k ffff88087ace0280 inode_cache 592 13644 15492 2582 4k kmem: dentry: partial list: slab: ffff880956591000 bad next pointer: 0 kmem: dentry: partial list: slab: ffff880956591000 bad prev pointer: 0 kmem: dentry: partial list: slab: ffff880956591000 bad inuse counter: 0 kmem: dentry: partial list: slab: ffff880956591000 bad s_mem pointer: 0 ffff88107bcc0240 dentry 192 25322 28820 1441 4k ffff88087bcc0240 names_cache 4096 0 14 14 4k kmem: radix_tree_node: full list: slab: ffff8809567df000 bad next pointer: 0 kmem: radix_tree_node: full list: slab: ffff8809567df000 bad prev pointer: 0 kmem: radix_tree_node: full list: slab: ffff8809567df000 bad inuse counter: 0 kmem: radix_tree_node: full list: slab: ffff8809567df000 bad s_mem pointer: 0 ffff88107bcb0200 radix_tree_node 560 17635 17675 2525 4k ffff88087bcb0200 key_jar 192 10 80 4 4k kmem: buffer_head: partial list: slab: ffff880956459000 bad next pointer: 0 kmem: buffer_head: partial list: slab: ffff880956459000 bad prev pointer: 0 kmem: buffer_head: partial list: slab: ffff880956459000 bad inuse counter: 0 kmem: buffer_head: partial list: slab: ffff880956459000 bad s_mem pointer: 0 ffff88107bca01c0 buffer_head 104 306911 1180448 31904 4k ffff88087bca01c0 nsproxy 48 0 0 0 4k ffff88107bc90180 vm_area_struct 200 3604 5662 298 4k ffff88087bc90180 mm_struct 1408 60 165 33 8k ffff88107bc80140 fs_cache 64 606 1593 27 4k ffff88087bc80140 files_cache 704 81 275 25 8k ffff88107fc70100 signal_cache 1088 1848 1974 282 8k ffff88087fc70100 sighand_cache 2112 1848 1902 634 8k ffff88107fc600c0 task_xstate 512 1198 1400 175 4k ffff88087fc600c0 task_struct 2624 1864 1926 642 8k ffff88107fc50080 cred_jar 192 1329 1960 98 4k ffff88087fc50080 anon_vma_chain 48 3011 7931 103 4k ffff88107fc40040 anon_vma 40 2218 4600 50 4k ffff88087fc40040 pid 128 1849 2340 78 4k ffff88047bf00a40 shared_policy_node 48 0 0 0 4k ffff88047fec0a00 numa_policy 136 31 56 2 4k ffff88047fe809c0 idr_layer_cache 544 425 588 84 4k ffff88047fe60980 size-4194304(DMA) 4194304 0 0 0 4096k ffff88047fe50940 size-4194304 4194304 0 0 0 4096k ffff88047fe40900 size-2097152(DMA) 2097152 0 0 0 2048k ffff88047fe308c0 size-2097152 2097152 0 0 0 2048k ffff88047fe20880 size-1048576(DMA) 1048576 0 0 0 1024k ffff88047fe10840 size-1048576 1048576 0 0 0 1024k ffff88047fe00800 size-524288(DMA) 524288 0 0 0 512k ffff88047fdf07c0 size-524288 524288 2 2 2 512k ffff88047fde0780 size-262144(DMA) 262144 0 0 0 256k ffff88047fdd0740 size-262144 262144 2 2 2 256k ffff88047fdc0700 size-131072(DMA) 131072 0 0 0 128k ffff88047fdb06c0 size-131072 131072 7 7 7 128k ffff88047fda0680 size-65536(DMA) 65536 0 0 0 64k ffff88047fd90640 size-65536 65536 5 5 5 64k ffff88047fd80600 size-32768(DMA) 32768 0 0 0 32k ffff88047fd705c0 size-32768 32768 3648 3648 3648 32k ffff88047fd60580 size-16384(DMA) 16384 0 0 0 16k ffff88047fd50540 size-16384 16384 136 159 159 16k ffff88047fd40500 size-8192(DMA) 8192 0 0 0 8k kmem: size-8192: full list: slab: ffff880aecdcf140 bad next pointer: 0 kmem: size-8192: full list: slab: ffff880aecdcf140 bad prev pointer: 0 kmem: size-8192: full list: slab: ffff880aecdcf140 bad inuse counter: 0 kmem: size-8192: full list: slab: ffff880aecdcf140 bad s_mem pointer: 0 ffff88047fd304c0 size-8192 8192 14663 14692 14692 8k ffff88047fd20480 size-4096(DMA) 4096 0 0 0 4k ffff88047fd10440 size-4096 4096 5354 5869 5869 4k ffff88047fd00400 size-2048(DMA) 2048 0 0 0 4k kmem: size-2048: full list: slab: ffff880955f25580 bad next pointer: 0 kmem: size-2048: full list: slab: ffff880955f25580 bad prev pointer: 0 kmem: size-2048: full list: slab: ffff880955f25580 bad inuse counter: 0 kmem: size-2048: full list: slab: ffff880955f25580 bad s_mem pointer: 0 ffff88047fcf03c0 size-2048 2048 11551 11576 5788 4k ffff88047fce0380 size-1024(DMA) 1024 0 0 0 4k kmem: size-1024: full list: slab: ffff880955f6fac0 bad next pointer: 0 kmem: size-1024: full list: slab: ffff880955f6fac0 bad prev pointer: 0 kmem: size-1024: full list: slab: ffff880955f6fac0 bad inuse counter: 0 kmem: size-1024: full list: slab: ffff880955f6fac0 bad s_mem pointer: 0 ffff88047fcd0340 size-1024 1024 7935 9008 2252 4k ffff88047fcc0300 size-512(DMA) 512 0 0 0 4k kmem: size-512: full list: slab: ffff880955ea02c0 bad next pointer: 0 kmem: size-512: full list: slab: ffff880955ea02c0 bad prev pointer: 0 kmem: size-512: full list: slab: ffff880955ea02c0 bad inuse counter: 0 kmem: size-512: full list: slab: ffff880955ea02c0 bad s_mem pointer: 0 ffff88047fcb02c0 size-512 512 54980 56512 7064 4k ffff88047fca0280 size-256(DMA) 256 0 0 0 4k kmem: size-256: full list: slab: ffff880aeccf7040 bad next pointer: 0 kmem: size-256: full list: slab: ffff880aeccf7040 bad prev pointer: 0 kmem: size-256: full list: slab: ffff880aeccf7040 bad inuse counter: 0 kmem: size-256: full list: slab: ffff880aeccf7040 bad s_mem pointer: 0 ffff88047fc90240 size-256 256 11481 14445 963 4k ffff88047fc80200 size-192(DMA) 192 0 0 0 4k kmem: size-192: full list: slab: ffff880955e3f040 bad next pointer: 0 kmem: size-192: full list: slab: ffff880955e3f040 bad prev pointer: 0 kmem: size-192: full list: slab: ffff880955e3f040 bad inuse counter: 0 kmem: size-192: full list: slab: ffff880955e3f040 bad s_mem pointer: 0 ffff88047fc701c0 size-192 192 76085 76280 3814 4k ffff88047fc60180 size-128(DMA) 128 0 0 0 4k ffff88047fc50140 size-64(DMA) 64 0 0 0 4k kmem: size-64: partial list: slab: ffff880955e40000 bad next pointer: 0 kmem: size-64: partial list: slab: ffff880955e40000 bad prev pointer: 0 kmem: size-64: partial list: slab: ffff880955e40000 bad inuse counter: 0 kmem: size-64: partial list: slab: ffff880955e40000 bad s_mem pointer: 0 ffff88047fc40100 size-64 64 55065 125257 2123 4k ffff88047fc300c0 size-32(DMA) 32 0 0 0 4k kmem: size-128: partial list: slab: ffff880955e2c000 bad next pointer: 0 kmem: size-128: partial list: slab: ffff880955e2c000 bad prev pointer: 0 kmem: size-128: partial list: slab: ffff880955e2c000 bad inuse counter: 0 kmem: size-128: partial list: slab: ffff880955e2c000 bad s_mem pointer: 0 ffff88047fc10080 size-128 128 56353 67980 2266 4k ffff88047fc00040 size-32 32 70114 84336 753 4k ffffffff81ac5a60 kmem_cache 32960 244 244 244 64k And pid:23161 Lbug is just a victim of the corruption, and thread 21945 could be the source : crash> bt 21945 PID: 21945 TASK: ffff880a5d88b540 CPU: 2 COMMAND: "mdt02_006" #0 [ffff88088e407e90] crash_nmi_callback at ffffffff81030096 #1 [ffff88088e407ea0] notifier_call_chain at ffffffff8152e6b5 #2 [ffff88088e407ee0] atomic_notifier_call_chain at ffffffff8152e71a #3 [ffff88088e407ef0] notify_die at ffffffff810a050e #4 [ffff88088e407f20] do_nmi at ffffffff8152c37b #5 [ffff88088e407f50] nmi at ffffffff8152bc40 [exception RIP: strncpy+16] RIP: ffffffff81289f70 RSP: ffff880a64257c50 RFLAGS: 00000286 RAX: ffff880aec345211 RBX: ffff880a7896f000 RCX: 0000000000000000 RDX: ffffffffff39dd53 RSI: ffff880a7bfb402a RDI: ffff880aecfa74ab RBP: ffff880a64257c50 R8: 0000000000000000 R9: 0000000000007c16 R10: 0000000000000000 R11: 9000000000000000 R12: ffff880aad914800 R13: ffff880aec345211 R14: ffff880aec345210 R15: 0000000000000000 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #6 [ffff880a64257c50] strncpy at ffffffff81289f70 #7 [ffff880a64257c58] mdt_get_info at ffffffffa06428b6 [mdt] #8 [ffff880a64257d58] mdt_handle_common at ffffffffa063bce7 [mdt] #9 [ffff880a64257da8] mds_regular_handle at ffffffffa0677cc5 [mdt] #10 [ffff880a64257db8] ptlrpc_server_handle_request at ffffffffa099c428 [ptlrpc] #11 [ffff880a64257eb8] ptlrpc_main at ffffffffa099d7be [ptlrpc] #12 [ffff880a64257f48] kernel_thread at ffffffff8100c20a strncpy(ffff880aecfa74ab,ffff880a7bfb402a,ffffffffff39dd53) .... 0xffffffffff39dd53 = -12985005 !!!! crash> bt -f 21945 PID: 21945 TASK: ffff880a5d88b540 CPU: 2 COMMAND: "mdt02_006" #0 [ffff88088e407e90] crash_nmi_callback at ffffffff81030096 .............. #5 [ffff88088e407f50] nmi at ffffffff8152bc40 [exception RIP: strncpy+16] RIP: ffffffff81289f70 RSP: ffff880a64257c50 RFLAGS: 00000286 RAX: ffff880aec345211 RBX: ffff880a7896f000 RCX: 0000000000000000 RDX: ffffffffff39dd53 RSI: ffff880a7bfb402a RDI: ffff880aecfa74ab RBP: ffff880a64257c50 R8: 0000000000000000 R9: 0000000000007c16 R10: 0000000000000000 R11: 9000000000000000 R12: ffff880aad914800 R13: ffff880aec345211 R14: ffff880aec345210 R15: 0000000000000000 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 --- --- #6 [ffff880a64257c50] strncpy at ffffffff81289f70 ffff880a64257c58: ffffffffa06428b6 #7 [ffff880a64257c58] mdt_get_info at ffffffffa06428b6 [mdt] ffff880a64257c60: 0000000000000006 ffff88047fc28e00 ffff880a64257c70: ffff880a64257d50 ffff880aec3441e8 ffff880a64257c80: ffff880a7896f488 ffff880aec344208 ffff880a64257c90: 0000000264257ce0 0000000000000246 ffff880a64257ca0: 0000000000000000 ffff8809da041168 ffff880a64257cb0: ffff88039c9079a0 ffff880a7896f2f0 ffff880a64257cc0: ffff880a7896f4e0 ffff88039c907000 ffff880a64257cd0: ffff880a7896f488 ffff880a7bfb4000 ffff880a64257ce0: 0000000064257d10 ffff880a97168800 ffff880a64257cf0: ffff880a7896f488 0000000000000000 ffff880a64257d00: 0000000000000000 0000000000000000 ffff880a64257d10: ffff880a64257d30 00000000a098cf0c ffff880a64257d20: ffffc9008ddd5538 ffff880a7896f000 ffff880a64257d30: ffff880a97168800 ffffffffa06a0e20 ffff880a64257d40: 0000000000000000 0000000000000000 ffff880a64257d50: ffff880a64257da0 ffffffffa063bce7 #8 [ffff880a64257d58] mdt_handle_common at ffffffffa063bce7 [mdt] ffff880a64257d60: ffff880a64257d80 ffffffffa098cc0c ffff880a64257d70: ffff880a64257db0 ffff880a97168800 ffff880a64257d80: ffff880ee10d2a80 ffff8808a3d91000 ffff880a64257d90: ffff8808a15dfdc0 ffff880a97168b40 ffff880a64257da0: ffff880a64257db0 ffffffffa0677cc5 #9 [ffff880a64257da8] mds_regular_handle at ffffffffa0677cc5 [mdt] ffff880a64257db0: ffff880a64257eb0 ffffffffa099c428 #10 [ffff880a64257db8] ptlrpc_server_handle_request at ffffffffa099c428 [ptlrpc] ffff880a64257dc0: ffff880a64257de0 ffff880a78c409c0 ffff880a64257dd0: 00000000000000c8 ffff8808a3d91068 ffff880a64257de0: ffff880ee10d2a80 ffff8808fa1e3600 ffff880a64257df0: ffff880a64257e00 ffffffffa049c5de ffff880a64257e00: ffff880a64257e20 ffffffffa04add9f ffff880a64257e10: ffff8808a15dfdc0 ffff8808a3d91000 ffff880a64257e20: ffff880a64257eb0 ffffffffa0993789 ffff880a64257e30: ffff8809342eeb40 ffff8808a3d91080 ffff880a64257e40: 0000000000000282 0000000000000000 ffff880a64257e50: 0000000000000001 000000000000001a ffff880a64257e60: ffff880a64257ea0 ffffffff81058bd3 ffff880a64257e70: 00000000537b0887 00000000000099ac ffff880a64257e80: ffff8808a3d91000 ffff8808a15dfdc0 ffff880a64257e90: ffff8808a3d91000 000000000000010e ffff880a64257ea0: ffff880ee10d2a80 ffff8808fa1e3600 ffff880a64257eb0: ffff880a64257f40 ffffffffa099d7be #11 [ffff880a64257eb8] ptlrpc_main at ffffffffa099d7be [ptlrpc] ffff880a64257ec0: ffff880800000000 ffff8808a3d91080 ffff880a64257ed0: 0000000000000000 ffff880ee10d2a80 ffff880a64257ee0: 0000003900000000 ffff8808a3d91068 ffff880a64257ef0: ffff8808a3d91048 ffff8808a15dfdf8 ffff880a64257f00: ffff8808fa1e3600 ffff8808a3d91030 ffff880a64257f10: 0000000000000000 ffff880f05736a80 ffff880a64257f20: 00000000ffffffef 0000000000000000 ffff880a64257f30: ffffffffa099ccf0 ffff8808a15dfdc0 ffff880a64257f40: ffff880f0573be20 ffffffff8100c20a #12 [ffff880a64257f48] kernel_thread at ffffffff8100c20a