crash> bt PID: 22785 TASK: ffff880612f90830 CPU: 1 COMMAND: "ll_ost_io_1011" #0 [ffff880612fd7620] machine_kexec at ffffffff8102902b #1 [ffff880612fd7680] crash_kexec at ffffffff810a5292 #2 [ffff880612fd7750] oops_end at ffffffff8149a050 #3 [ffff880612fd7780] die at ffffffff8100714b #4 [ffff880612fd77b0] do_general_protection at ffffffff81499be2 #5 [ffff880612fd77e0] general_protection at ffffffff814993b5 [exception RIP: radix_tree_lookup_slot+5] RIP: ffffffff81261465 RSP: ffff880612fd7890 RFLAGS: 00010286 RAX: e940201000000010 RBX: e940201000000008 RCX: 0000000000000000 RDX: 00000000000200d2 RSI: 0000000000000000 RDI: e940201000000008 RBP: ffff880612fd78b0 R8: ffff880612fdc140 R9: 0000000000000008 R10: 0000000000001000 R11: 0000000000000001 R12: 0000000000000000 R13: 0000000000000000 R14: e940201000000000 R15: 20105fa000080221 ORIG_RAX: ffffffffffffffff CS: 0010 SS: 0018 #6 [ffff880612fd7898] find_get_page at ffffffff810ffe8e #7 [ffff880612fd78b8] find_lock_page at ffffffff8110112a #8 [ffff880612fd78e8] find_or_create_page at ffffffff8110129f #9 [ffff880612fd7938] filter_get_page at ffffffffa0c4b065 [obdfilter] #10 [ffff880612fd7968] filter_preprw_read at ffffffffa0c4d64d [obdfilter] #11 [ffff880612fd7a98] filter_preprw at ffffffffa0c4dedc [obdfilter] #12 [ffff880612fd7ad8] obd_preprw at ffffffffa0c09051 [ost] #13 [ffff880612fd7b48] ost_brw_read at ffffffffa0c10091 [ost] #14 [ffff880612fd7c88] ost_handle at ffffffffa0c16423 [ost] #15 [ffff880612fd7da8] ptlrpc_main at ffffffffa07fd4e6 [ptlrpc] #16 [ffff880612fd7f48] kernel_thread at ffffffff8100412a crash> dis radix_tree_lookup_slot 0xffffffff81261460 : push %rbp 0xffffffff81261461 : lea 0x8(%rdi),%rax 0xffffffff81261465 : mov 0x8(%rdi),%rdi # dereference of pointer in 1st argument of radix_tree_lookup_slot 0xffffffff81261469 : mov %rsp,%rbp RDI contains an invalid address: RDI: e940201000000008 RDI comes from mapping->page_tree in find_get_page(): struct page *find_get_page(struct address_space *mapping, pgoff_t offset) { void **pagep; struct page *page; rcu_read_lock(); repeat: page = NULL; pagep = radix_tree_lookup_slot(&mapping->page_tree, offset); ... mapping is itself passed directly from find_lock_page(): struct page *find_lock_page(struct address_space *mapping, pgoff_t offset) { struct page *page; repeat: page = find_get_page(mapping, offset); ... Also passed directly from find_or_create_page(): struct page *find_or_create_page(struct address_space *mapping, pgoff_t index, gfp_t gfp_mask) { struct page *page; int err; repeat: page = find_lock_page(mapping, index); ... Passed from the mapping field of an inode structure in filter_get_page(): static struct page *filter_get_page(struct obd_device *obd, struct inode *inode, obd_off offset, int localreq) { struct page *page; page = find_or_create_page(inode->i_mapping, offset >> CFS_PAGE_SHIFT, (localreq ? (GFP_NOFS | __GFP_HIGHMEM) : GFP_HIGHUSER)); The inode address is the second argument of filter_get_page() passed in filter_preprw_read(): 0xffffffffa0c4d63c : mov -0xd0(%rbp),%rsi # rsi is the second argument for the next call. # inode address is stored in %rbp (of filter_preprw_read) - 0xd0 0xffffffffa0c4d643 : xor %ecx,%ecx 0xffffffffa0c4d645 : mov %rbx,%rdi 0xffffffffa0c4d648 : callq 0xffffffffa0c4b030 #10 [ffff880612fd7968] filter_preprw_read at ffffffffa0c4d64d [obdfilter] ffff880612fd7970: 00050000c0a800c8 ffff880200000050 ffff880612fd7980: ffff88046ac3c540 ffff880610f485c8 ffff880612fd7990: ffff880610f482e8 ffff8805235be548 ffff880612fd79a0: 0000000152390c89 ffff88060b9debc0 ffff880612fd79b0: ffff880bb40df400 ffff88045aff22c0 ffff880612fd79c0: ffff880040aa57c0 ffff880500000000 ffff880612fd79d0: ffff880c30c72580 ffff880c309cc3c0 ffff880612fd79e0: 00007ffffffff000 ffffffffa082d6bb ffff880612fd79f0: 0000000000000001 0000000000000298 ffff880612fd7a00: ffff880600000012 ffff880599f6dc00 ffff880612fd7a10: ffffffffa08990a0 0000000000000000 ffff880612fd7a20: ffff880599f6dc64 0000000000000000 ffff880612fd7a30: ffff8806c0ffeeaa ffffffffa081a414 ffff880612fd7a40: ffff880091cab108 0000000000000003 ffff880612fd7a50: 00000000531db489 0000000000073e70 ffff880612fd7a60: ffff880599f6dc00 0000000000000000 ffff880612fd7a70: 0000000000000001 ffff8805235be530 ffff880612fd7a80: ffff880091cab1c8 ffff8805235be548 ffff880612fd7a90: ffff880612fd7ad0 ffffffffa0c4dedc filter_preprw_read base pointer is ffff880612fd7a90. crash> p/x 0xffff880612fd7a90-0xd0 $64 = 0xffff880612fd79c0 crash> rd 0xffff880612fd79c0 ffff880612fd79c0: ffff880040aa57c0 .W.@.... inode struct is stored at ffff880040aa57c0. Let's dump it: crash> struct inode ffff880040aa57c0 struct inode { i_hash = { next = 0x41f0000040f, pprev = 0x201080000000202f }, i_list = { next = 0x70000, prev = 0x448e201000000000 }, i_sb_list = { next = 0x42000000410, prev = 0x2010800000002230 }, i_dentry = { next = 0x70000, prev = 0xb68f201000000000 }, i_ino = 2251868533686272, i_count = { counter = 524321 }, i_nlink = 465788784, i_uid = 262236, i_gid = 0, i_rdev = 0, i_version = 2251872828653569, i_size = 2310451749640995361, i_atime = { tv_sec = 1394455689, tv_nsec = 4810161130099965952 }, i_mtime = { tv_sec = 2251877123620866, tv_nsec = 2310487346329945122 }, i_ctime = { tv_sec = 458752, tv_nsec = 1832718826454646784 }, i_blocks = 2251881418588163, i_blkbits = 525859, i_bytes = 32768, i_mode = 8208, i_lock = { raw_lock = { slock = 458752 } }, i_mutex = { count = { counter = 0 }, wait_lock = { raw_lock = { slock = 1969496080 } }, wait_list = { next = 0x8001400080004, prev = 0x2010800000080824 }, owner = 0x70000 }, i_alloc_sem = { count = 3555064208947150848, wait_lock = { raw_lock = { slock = 524293 } }, wait_list = { next = 0x2010800000080a25, prev = 0x70000 } }, i_op = 0x5d5d201000000000, i_fop = 0x8001600080006, i_sb = 0x2010800000080c26, i_flock = 0x70000, i_mapping = 0xe940201000000000, i_data = { host = 0x8001700080007, page_tree = { height = 527911, gfp_mask = 537952256, rnode = 0x70000 }, tree_lock = { raw_lock = { slock = 0 } }, i_mmap_writable = 2236293136, i_mmap = { prio_tree_node = 0x8001800080008, index_bits = 4136, raw = 8 }, i_mmap_nonlinear = { next = 0x70000, prev = 0x6124201000000000 }, i_mmap_lock = { raw_lock = { slock = 524297 } }, truncate_count = 524313, nrpages = 2310482943988470313, writeback_index = 327680, a_ops = 0xefe5201000000000, flags = 2251911483359242, backing_dev_info = 0x201080000008142a, private_lock = { raw_lock = { slock = 458752 } }, private_list = { next = 0xb932201000000000, prev = 0x8001b0008000b }, assoc_mapping = 0x20107bff0008162b }, i_dquot = {0x50000, 0x37f3201000000000}, i_devices = { next = 0x8001c0008000c, prev = 0x201080000008182c }, { i_pipe = 0x70000, i_bdev = 0x70000, i_cdev = 0x70000 }, i_generation = 0, i_fsnotify_mask = 2433425424, i_fsnotify_mark_entries = { first = 0x8001d0008000d }, inotify_watches = { next = 0x2010800000081a2d, prev = 0x70000 }, inotify_mutex = { count = { counter = 0 }, wait_lock = { raw_lock = { slock = 4244643856 } }, wait_list = { next = 0x8001e0008000e, prev = 0x2010800000081c2e }, owner = 0x70000 }, i_state = 5268402392184913920, dirtied_when = 2251932958195727, i_flags = 532015, i_writecount = { counter = 537952256 }, i_security = 0x70000, i_acl = 0x2516201000000000, i_default_acl = 0x10001000100000, i_private = 0x16165eb800100020 } The only information which seems to be consistent with an inode is the access time: i_atime = { tv_sec = 1394455689, tv_nsec = 4810161130099965952 }, spiechurski@oban:/dumps/files/140310-BS0206$ date -d @1394455689 Mon Mar 10 13:48:09 CET 2014 which is exactly the time of the crash: crash> sys SYSTEM MAP: /dumps/lib/kernel-debuginfo/2.6.32-279.5.2.bl6.Bull.36.x86_64/boot/System.map-2.6.32-279.5.2.bl6.Bull.36.x86_64 DEBUG KERNEL: /dumps/lib/kernel-debuginfo/2.6.32-279.5.2.bl6.Bull.36.x86_64/modules/vmlinux (2.6.32-279.5.2.bl6.Bull.36.x86_64) DUMPFILE: vmcore [PARTIAL DUMP] CPUS: 8 DATE: Mon Mar 10 13:48:09 2014 ... There seems to be a pattern in the memory region: crash> rd -x ffff880040aa57c0 592 ffff880040aa57c0: 0000041f0000040f 201080000000202f ffff880040aa57d0: 0000000000070000 448e201000000000 ffff880040aa57e0: 0000042000000410 2010800000002230 ffff880040aa57f0: 0000000000070000 b68f201000000000 ffff880040aa5800: 0008001000080000 1bc35f7000080021 ffff880040aa5810: 000000000004005c 45281bc200000000 ffff880040aa5820: 0008001100080001 20105fa000080221 ffff880040aa5830: 00000000531db489 42c1201000000000 ffff880040aa5840: 0008001200080002 2010800000080422 ffff880040aa5850: 0000000000070000 196f201000000000 ffff880040aa5860: 0008001300080003 2010800000080623 ffff880040aa5870: 0000000000070000 7564201000000000 ffff880040aa5880: 0008001400080004 2010800000080824 ffff880040aa5890: 0000000000070000 3156201000000000 ffff880040aa58a0: 0008001500080005 2010800000080a25 ffff880040aa58b0: 0000000000070000 5d5d201000000000 ffff880040aa58c0: 0008001600080006 2010800000080c26 ffff880040aa58d0: 0000000000070000 e940201000000000 ffff880040aa58e0: 0008001700080007 2010800000080e27 ffff880040aa58f0: 0000000000070000 854b201000000000 ffff880040aa5900: 0008001800080008 2010800000081028 ffff880040aa5910: 0000000000070000 6124201000000000 ffff880040aa5920: 0008001900080009 20107bff00081229 ffff880040aa5930: 0000000000050000 efe5201000000000 ffff880040aa5940: 0008001a0008000a 201080000008142a ffff880040aa5950: 0000000000070000 b932201000000000 ffff880040aa5960: 0008001b0008000b 20107bff0008162b ffff880040aa5970: 0000000000050000 37f3201000000000 ffff880040aa5980: 0008001c0008000c 201080000008182c ffff880040aa5990: 0000000000070000 910b201000000000 ffff880040aa59a0: 0008001d0008000d 2010800000081a2d ffff880040aa59b0: 0000000000070000 fd00201000000000 ffff880040aa59c0: 0008001e0008000e 2010800000081c2e ffff880040aa59d0: 0000000000070000 491d201000000000 ffff880040aa59e0: 0008001f0008000f 2010800000081e2f ffff880040aa59f0: 0000000000070000 2516201000000000 ffff880040aa5a00: 0010001000100000 16165eb800100020 ffff880040aa5a10: 000000000004010d 0162161600000000 ffff880040aa5a20: 0010001100100001 2010540f00100221 ffff880040aa5a30: 0000000000050000 d09d201000000000 ffff880040aa5a40: 0010001200100002 2010800000100422 ffff880040aa5a50: 0000000000070000 8c55201000000000 ffff880040aa5a60: 0010001300100003 2010800000100623 ffff880040aa5a70: 0000000000070000 e05e201000000000 ffff880040aa5a80: 0010001400100004 2010800000100824 ffff880040aa5a90: 0000000000070000 a46c201000000000 ffff880040aa5aa0: 0010001500100005 2010800000100a25 ffff880040aa5ab0: 0000000000070000 c867201000000000 ffff880040aa5ac0: 0010001600100006 2010800000100c26 ffff880040aa5ad0: 0000000000070000 7c7a201000000000 ffff880040aa5ae0: 0010001700100007 2010800000100e27 ffff880040aa5af0: 0000000000070000 1071201000000000 ffff880040aa5b00: 0010001800100008 2010800000101028 ffff880040aa5b10: 0000000000070000 f41e201000000000 ffff880040aa5b20: 0010001900100009 2010800000101229 ffff880040aa5b30: 0000000000070000 9815201000000000 ffff880040aa5b40: 0010001a0010000a 201080000010142a ffff880040aa5b50: 0000000000070000 2c08201000000000 ffff880040aa5b60: 0010001b0010000b 201080000010162b ffff880040aa5b70: 0000000000070000 4003201000000000 ffff880040aa5b80: 0010001c0010000c 201080000010182c ffff880040aa5b90: 0000000000070000 0431201000000000 ffff880040aa5ba0: 0010001d0010000d 2010800000101a2d ffff880040aa5bb0: 0000000000070000 683a201000000000 ffff880040aa5bc0: 0010001e0010000e 2010800000101c2e ffff880040aa5bd0: 0000000000070000 dc27201000000000 ffff880040aa5be0: 0010001f0010000f 2010800000101e2f ffff880040aa5bf0: 0000000000070000 b02c201000000000 ffff880040aa5c00: 0018001000180000 14425bfe00180020