== sanity test 101a: check read-ahead for random reads =============================================== 23:05:47 (1400133947) debug=-1 nreads: 10000 file size: 96MB = backtrace = This is the loop the client is in: while (!atomic64_add_unless(cli->cl_lru_left, -1, 0)) { /* run out of LRU spaces, try to drop some by itself */ rc = osc_lru_reclaim(cli); if (rc < 0) break; if (rc > 0) continue; cond_resched(); rc = l_wait_event(osc_lru_waitq, atomic64_read(cli->cl_lru_left) > 0, &lwi); if (rc < 0) break; } == bt == crash> bt 22025 PID: 22025 TASK: ffff88017aeba480 CPU: 5 COMMAND: "reads" #0 [ffff88017b2ff838] schedule at ffffffff8145ec7b #1 [ffff88017b2ff980] osc_lru_reserve at ffffffffa0e16ee5 [osc] #2 [ffff88017b2ffa00] osc_page_init at ffffffffa0e1710d [osc] #3 [ffff88017b2ffa40] lov_page_init_raid0 at ffffffffa0ea48b0 [lov] #4 [ffff88017b2ffaa0] cl_page_alloc at ffffffffa0aae632 [obdclass] #5 [ffff88017b2ffae0] cl_page_find at ffffffffa0aae91b [obdclass] #6 [ffff88017b2ffb30] ll_write_begin at ffffffffa0f96f8d [lustre] #7 [ffff88017b2ffb90] generic_perform_write at ffffffff810f8242 #8 [ffff88017b2ffc10] generic_file_buffered_write at ffffffff810f83a1 #9 [ffff88017b2ffc60] __generic_file_aio_write at ffffffff810fb336 #10 [ffff88017b2ffd10] generic_file_aio_write at ffffffff810fb57c #11 [ffff88017b2ffd50] vvp_io_write_start at ffffffffa0faae48 [lustre] #12 [ffff88017b2ffda0] cl_io_start at ffffffffa0ab65f9 [obdclass] #13 [ffff88017b2ffdd0] cl_io_loop at ffffffffa0aba123 [obdclass] #14 [ffff88017b2ffe00] ll_file_io_generic at ffffffffa0f46af1 [lustre] #15 [ffff88017b2ffe70] ll_file_aio_write at ffffffffa0f47037 [lustre] #16 [ffff88017b2ffec0] ll_file_write at ffffffffa0f47a00 [lustre] #17 [ffff88017b2fff10] vfs_write at ffffffff8115aeae #18 [ffff88017b2fff40] sys_write at ffffffff8115b023 #19 [ffff88017b2fff80] system_call_fastpath at ffffffff81468d92 RIP: 00002aaaaad99630 RSP: 00007fffffffc568 RFLAGS: 00010246 RAX: 0000000000000001 RBX: ffffffff81468d92 RCX: 00007fffffffc510 RDX: 0000000000010000 RSI: 0000000000603040 RDI: 0000000000000003 RBP: 0000000000010000 R8: 0000000000000000 R9: 0101010101010101 R10: 00007fffffffc3b0 R11: 0000000000000246 R12: 0000000000010000 R13: 0000000000000001 R14: 00000000063b0000 R15: 00000000063c0000 ORIG_RAX: 0000000000000001 CS: 0033 SS: 002b == bt -f == crash> bt -f 22025 PID: 22025 TASK: ffff88017aeba480 CPU: 5 COMMAND: "reads" #0 [ffff88017b2ff838] schedule at ffffffff8145ec7b ffff88017b2ff840: 0000000000000086 ffff88017b2fe010 ffff88017b2ff850: 0000000000011800 0000000000011800 ffff88017b2ff860: 0000000000011800 0000000000011800 ffff88017b2ff870: ffff88017b2fffd8 ffff88017b2fffd8 ffff88017b2ff880: 0000000000011800 ffff88017aeba480 ffff88017b2ff890: ffff880233238040 0000000000000000 ffff88017b2ff8a0: ffff880155dbc0e0 0000000000000000 ffff88017b2ff8b0: ffff880155dbc0e8 ffff880155dbc0f8 ffff88017b2ff8c0: ffffffffa09dfeef ffff880100000010 ffff88017b2ff8d0: ffff88017b2ff928 ffff88017b2ff8e8 ffff88017b2ff8e0: 0000000000000000 ffff880155dbc0e8 ffff88017b2ff8f0: ffffffffa09e873e ffff880430d09084 ffff88017b2ff900: ffff880430d09538 0000000000000000 ffff88017b2ff910: 0000000000000004 0000000000000000 ffff88017b2ff920: ffffffffa0e168f2 ffff880100000010 ffff88017b2ff930: ffff880179bdee60 ffff880100000001 ffff88017b2ff940: ffff880176e07e58 0000000000000286 ffff88017b2ff950: 0000000000000000 ffff88036bf6c160 ffff88017b2ff960: ffff880431ea5538 ffff88017aeba480 ffff88017b2ff970: ffff88017aeba480 ffff880431ea5078 ffff88017b2ff980: ffffffffa0e16ee5 #1 [ffff88017b2ff980] osc_lru_reserve at ffffffffa0e16ee5 [osc] ffff88017b2ff988: ffff88017aeba480 0000000000000000 ffff88017b2ff998: ffff880400000000 ffff88017aeba480 ffff88017b2ff9a8: ffffffff81054b50 ffffffffa0e47a58 ffff88017b2ff9b8: ffffffffa0e47a58 0000000000000000 ffff88017b2ff9c8: ffff88036bf6c160 ffff88036bf6c160 ffff88017b2ff9d8: 0000000000000000 ffff880352ecfc48 ffff88017b2ff9e8: ffff88036bf6c000 000000000000001e ffff88017b2ff9f8: ffff880432669290 ffffffffa0e1710d #2 [ffff88017b2ffa00] osc_page_init at ffffffffa0e1710d [osc] ffff88017b2ffa08: ffff88036aad2278 ffffffffa0e16fb0 ffff88017b2ffa18: ffff880352ecfc48 ffff88036aad2200 ffff88017b2ffa28: ffff88037fc7f4c0 ffff88036aad2278 ffff88017b2ffa38: ffff88036bf6c000 ffffffffa0ea48b0 == bt -F == crash> bt -F 22025 PID: 22025 TASK: ffff88017aeba480 CPU: 5 COMMAND: "reads" #0 [ffff88017b2ff838] schedule at ffffffff8145ec7b ffff88017b2ff840: 0000000000000086 ffff88017b2fe010 ffff88017b2ff850: 0000000000011800 0000000000011800 ffff88017b2ff860: 0000000000011800 0000000000011800 ffff88017b2ff870: ffff88017b2fffd8 ffff88017b2fffd8 ffff88017b2ff880: 0000000000011800 [task_struct] ffff88017b2ff890: [task_struct] 0000000000000000 ffff88017b2ff8a0: [size-16384] 0000000000000000 ffff88017b2ff8b0: [size-16384] [size-16384] ffff88017b2ff8c0: libcfs_debug_msg+63 ffff880100000010 ffff88017b2ff8d0: ffff88017b2ff928 ffff88017b2ff8e8 ffff88017b2ff8e0: 0000000000000000 [size-16384] ffff88017b2ff8f0: cfs_hash_add_unique+14 [ll_obd_dev_cache] ffff88017b2ff900: [ll_obd_dev_cache] 0000000000000000 ffff88017b2ff910: 0000000000000004 0000000000000000 ffff88017b2ff920: osc_lru_reclaim+514 ffff880100000010 ffff88017b2ff930: [cl_env_kmem] ffff880100000001 ffff88017b2ff940: [cl_env_kmem] 0000000000000286 ffff88017b2ff950: 0000000000000000 [size-1024] ffff88017b2ff960: [ll_obd_dev_cache] [task_struct] ffff88017b2ff970: [task_struct] [ll_obd_dev_cache] ffff88017b2ff980: osc_lru_reserve+693 #1 [ffff88017b2ff980] osc_lru_reserve at ffffffffa0e16ee5 [osc] ffff88017b2ff988: [task_struct] 0000000000000000 ffff88017b2ff998: ffff880400000000 [task_struct] ffff88017b2ff9a8: default_wake_function osc_lru_waitq+8 ffff88017b2ff9b8: osc_lru_waitq+8 0000000000000000 ffff88017b2ff9c8: [size-1024] [size-1024] ffff88017b2ff9d8: 0000000000000000 [osc_object_kmem] ffff88017b2ff9e8: [size-1024] 000000000000001e ffff88017b2ff9f8: [cl_env_kmem] osc_page_init+349 #2 [ffff88017b2ffa00] osc_page_init at ffffffffa0e1710d [osc] ffff88017b2ffa08: [lovsub_object_kmem] osc_page_init ffff88017b2ffa18: [osc_object_kmem] [lovsub_object_kmem] ffff88017b2ffa28: [size-128] [lovsub_object_kmem] ffff88017b2ffa38: [size-1024] lov_page_init_raid0+384 = disassemble = == osc_lru_reserve == crash> dis osc_lru_reserve | sed -e 's/^/ /' 0xffffffffa0e16c30 : push %r15 0xffffffffa0e16c32 : push %r14 0xffffffffa0e16c34 : push %r13 0xffffffffa0e16c36 : push %r12 0xffffffffa0e16c38 : mov %rdx,%r12 0xffffffffa0e16c3b : push %rbp 0xffffffffa0e16c3c : mov %rsi,%rbp 0xffffffffa0e16c3f : mov $0xffffffffa0e471a0,%rsi 0xffffffffa0e16c46 : push %rbx 0xffffffffa0e16c47 : sub $0x48,%rsp 0xffffffffa0e16c4b : mov 0x30(%rdi),%rdi 0xffffffffa0e16c4f : callq 0xffffffffa0a9d5c0 0xffffffffa0e16c54 : test %rax,%rax 0xffffffffa0e16c57 : mov %rax,%rbx 0xffffffffa0e16c5a : je 0xffffffffa0e16f4c 0xffffffffa0e16c60 : mov 0x8(%rbp),%rax 0xffffffffa0e16c64 : testb $0x1,-0x4127c7(%rip) # 0xffffffffa0a044a4 0xffffffffa0e16c6b : mov 0x48(%rax),%rax 0xffffffffa0e16c6f : mov 0xc0(%rax),%rbp 0xffffffffa0e16c76 : lea 0x4c0(%rbp),%r13 0xffffffffa0e16c7d : jne 0xffffffffa0e16cd0 0xffffffffa0e16c7f : cmpq $0x0,0xc50(%rbp) 0xffffffffa0e16c87 : je 0xffffffffa0e16d3b 0xffffffffa0e16c8d : mov 0x40(%rbx),%eax 0xffffffffa0e16c90 : test %eax,%eax 0xffffffffa0e16c92 : jle 0xffffffffa0e16dc0 0xffffffffa0e16c98 : sub $0x1,%eax 0xffffffffa0e16c9b : mov %eax,0x40(%rbx) 0xffffffffa0e16c9e : lock incq 0xc70(%rbp) 0xffffffffa0e16ca6 : orb $0x4,0xb0(%r12) 0xffffffffa0e16caf : xor %ebx,%ebx 0xffffffffa0e16cb1 : testb $0x1,-0x412814(%rip) # 0xffffffffa0a044a4 0xffffffffa0e16cb8 : jne 0xffffffffa0e16d70 0xffffffffa0e16cbe : mov %ebx,%eax 0xffffffffa0e16cc0 : add $0x48,%rsp 0xffffffffa0e16cc4 : pop %rbx 0xffffffffa0e16cc5 : pop %rbp 0xffffffffa0e16cc6 : pop %r12 0xffffffffa0e16cc8 : pop %r13 0xffffffffa0e16cca : pop %r14 0xffffffffa0e16ccc : pop %r15 0xffffffffa0e16cce : retq 0xffffffffa0e16ccf : nop 0xffffffffa0e16cd0 : testb $0x8,-0x412837(%rip) # 0xffffffffa0a044a0 0xffffffffa0e16cd7 : je 0xffffffffa0e16c7f 0xffffffffa0e16cd9 : xor %eax,%eax 0xffffffffa0e16cdb : mov $0xffffffffa0e3bf1b,%rsi 0xffffffffa0e16ce2 : mov $0xffffffffa0e50700,%rdi 0xffffffffa0e16ce9 : movl $0x8,0x39a1d(%rip) # 0xffffffffa0e50710 0xffffffffa0e16cf3 : movq $0xffffffffa0e37f30,0x39a02(%rip) # 0xffffffffa0e50700 0xffffffffa0e16cfe : movq $0xffffffffa0e34d50,0x399ff(%rip) # 0xffffffffa0e50708 0xffffffffa0e16d09 : movl $0x36e,0x39a01(%rip) # 0xffffffffa0e50714 0xffffffffa0e16d13 : movq $0x0,0x39a02(%rip) # 0xffffffffa0e50720 0xffffffffa0e16d1e : movl $0x1,0x399f0(%rip) # 0xffffffffa0e50718 0xffffffffa0e16d28 : callq 0xffffffffa09dfeb0 0xffffffffa0e16d2d : cmpq $0x0,0xc50(%rbp) 0xffffffffa0e16d35 : jne 0xffffffffa0e16c8d 0xffffffffa0e16d3b : testb $0x1,-0x41289e(%rip) # 0xffffffffa0a044a4 0xffffffffa0e16d42 : je 0xffffffffa0e16da8 0xffffffffa0e16d44 : testb $0x8,-0x4128ab(%rip) # 0xffffffffa0a044a0 0xffffffffa0e16d4b : je 0xffffffffa0e16da8 0xffffffffa0e16d4d : xor %esi,%esi 0xffffffffa0e16d4f : mov $0xffffffffa0e48240,%rdi 0xffffffffa0e16d56 : movl $0x1,0x314f8(%rip) # 0xffffffffa0e48258 0xffffffffa0e16d60 : callq 0xffffffffa09da240 0xffffffffa0e16d65 : jmpq 0xffffffffa0e16cc0 0xffffffffa0e16d6a : nopw 0x0(%rax,%rax,1) 0xffffffffa0e16d70 : testb $0x8,-0x4128d7(%rip) # 0xffffffffa0a044a0 0xffffffffa0e16d77 : je 0xffffffffa0e16cbe 0xffffffffa0e16d7d : mov %rbx,%rsi 0xffffffffa0e16d80 : mov $0xffffffffa0e481c0,%rdi 0xffffffffa0e16d87 : movl $0x1,0x31447(%rip) # 0xffffffffa0e481d8 0xffffffffa0e16d91 : callq 0xffffffffa09da240 0xffffffffa0e16d96 : add $0x48,%rsp 0xffffffffa0e16d9a : pop %rbx 0xffffffffa0e16d9b : pop %rbp 0xffffffffa0e16d9c : pop %r12 0xffffffffa0e16d9e : pop %r13 0xffffffffa0e16da0 : pop %r14 0xffffffffa0e16da2 : pop %r15 0xffffffffa0e16da4 : retq 0xffffffffa0e16da5 : nopl (%rax) 0xffffffffa0e16da8 : add $0x48,%rsp 0xffffffffa0e16dac : xor %eax,%eax 0xffffffffa0e16dae : pop %rbx 0xffffffffa0e16daf : pop %rbp 0xffffffffa0e16db0 : pop %r12 0xffffffffa0e16db2 : pop %r13 0xffffffffa0e16db4 : pop %r14 0xffffffffa0e16db6 : pop %r15 0xffffffffa0e16db8 : retq 0xffffffffa0e16db9 : nopl 0x0(%rax) 0xffffffffa0e16dc0 : mov 0xc68(%rbp),%rsi 0xffffffffa0e16dc7 : mov (%rsi),%rax 0xffffffffa0e16dca : test %rax,%rax 0xffffffffa0e16dcd : js 0xffffffffa0e16f7e 0xffffffffa0e16dd3 : mov %gs:0xb600,%r14 0xffffffffa0e16ddc : xor %ebx,%ebx 0xffffffffa0e16dde : mov %r14,%r15 0xffffffffa0e16de1 : mov %r14,(%rsp) 0xffffffffa0e16de5 : nopl (%rax) 0xffffffffa0e16de8 : mov (%rsi),%rdx 0xffffffffa0e16deb : test %rdx,%rdx 0xffffffffa0e16dee : je 0xffffffffa0e16e40 0xffffffffa0e16df0 : lea -0x1(%rdx),%rdi 0xffffffffa0e16df4 : mov %rdx,%rax 0xffffffffa0e16df7 : lock cmpxchg %rdi,(%rsi) 0xffffffffa0e16dfc : cmp %rax,%rdx 0xffffffffa0e16dff : mov %rax,%rcx 0xffffffffa0e16e02 : jne 0xffffffffa0e16e13 0xffffffffa0e16e04 : jmp 0xffffffffa0e16e29 0xffffffffa0e16e06 : nopw %cs:0x0(%rax,%rax,1) 0xffffffffa0e16e10 : mov %rax,%rcx 0xffffffffa0e16e13 : test %rcx,%rcx 0xffffffffa0e16e16 : je 0xffffffffa0e16e40 0xffffffffa0e16e18 : lea -0x1(%rcx),%rdi 0xffffffffa0e16e1c : mov %rcx,%rax 0xffffffffa0e16e1f : lock cmpxchg %rdi,(%rsi) 0xffffffffa0e16e24 : cmp %rcx,%rax 0xffffffffa0e16e27 : jne 0xffffffffa0e16e10 0xffffffffa0e16e29 : test %rbx,%rbx 0xffffffffa0e16e2c : js 0xffffffffa0e16cb1 0xffffffffa0e16e32 : jmpq 0xffffffffa0e16c9e 0xffffffffa0e16e37 : nopw 0x0(%rax,%rax,1) 0xffffffffa0e16e40 : mov %r13,%rdi 0xffffffffa0e16e43 : nopl 0x0(%rax,%rax,1) 0xffffffffa0e16e48 : callq 0xffffffffa0e166f0 0xffffffffa0e16e4d : cmp $0x0,%rax 0xffffffffa0e16e51 : mov %rax,%rbx 0xffffffffa0e16e54 : jl 0xffffffffa0e16cb1 0xffffffffa0e16e5a : je 0xffffffffa0e16e68 0xffffffffa0e16e5c : mov 0xc68(%rbp),%rsi 0xffffffffa0e16e63 : jmp 0xffffffffa0e16de8 0xffffffffa0e16e65 : nopl (%rax) 0xffffffffa0e16e68 : callq 0xffffffff8145ef70 <_cond_resched> 0xffffffffa0e16e6d : mov 0xc68(%rbp),%rsi 0xffffffffa0e16e74 : mov (%rsi),%rax 0xffffffffa0e16e77 : test %rax,%rax 0xffffffffa0e16e7a : jg 0xffffffffa0e16de8 0xffffffffa0e16e80 : lea 0x10(%rsp),%rsi 0xffffffffa0e16e85 : mov $0xffffffffa0e47a50,%rdi 0xffffffffa0e16e8c : movl $0x0,0x10(%rsp) 0xffffffffa0e16e94 : mov %r14,0x18(%rsp) 0xffffffffa0e16e99 : movq $0xffffffff81054b50,0x20(%rsp) 0xffffffffa0e16ea2 : callq 0xffffffff81083320 0xffffffffa0e16ea7 : mov $0x6106,%edi 0xffffffffa0e16eac : callq 0xffffffffa09d01c0 0xffffffffa0e16eb1 : mov %rax,0x8(%rsp) 0xffffffffa0e16eb6 : movq $0x1,0x38(%rsp) 0xffffffffa0e16ebf : mov 0x38(%rsp),%rax 0xffffffffa0e16ec4 : xchg %rax,(%r15) 0xffffffffa0e16ec7 : mov %rax,0x38(%rsp) 0xffffffffa0e16ecc : mov 0x38(%rsp),%rax 0xffffffffa0e16ed1 : mov 0xc68(%rbp),%rax 0xffffffffa0e16ed8 : mov (%rax),%rax 0xffffffffa0e16edb : test %rax,%rax 0xffffffffa0e16ede : jg 0xffffffffa0e16f04 0xffffffffa0e16ee0 : callq 0xffffffff8145efb0 0xffffffffa0e16ee5 : mov 0xc68(%rbp),%rax 0xffffffffa0e16eec : mov (%rax),%rax 0xffffffffa0e16eef : test %rax,%rax 0xffffffffa0e16ef2 : jg 0xffffffffa0e16f04 0xffffffffa0e16ef4 : callq 0xffffffffa09d00e0 0xffffffffa0e16ef9 : test %eax,%eax 0xffffffffa0e16efb : je 0xffffffffa0e16eb6 0xffffffffa0e16efd : mov $0xfffffffffffffffc,%rbx 0xffffffffa0e16f04 : mov 0x8(%rsp),%rdi 0xffffffffa0e16f09 : callq 0xffffffffa09d0150 0xffffffffa0e16f0e : movq $0x0,0x38(%rsp) 0xffffffffa0e16f17 : mov (%rsp),%rdx 0xffffffffa0e16f1b : mov 0x38(%rsp),%rax 0xffffffffa0e16f20 : xchg %rax,(%rdx) 0xffffffffa0e16f23 : lea 0x10(%rsp),%rsi 0xffffffffa0e16f28 : mov $0xffffffffa0e47a50,%rdi 0xffffffffa0e16f2f : mov %rax,0x38(%rsp) 0xffffffffa0e16f34 : mov 0x38(%rsp),%rax 0xffffffffa0e16f39 : callq 0xffffffff81083260 0xffffffffa0e16f3e : test %rbx,%rbx 0xffffffffa0e16f41 : jne 0xffffffffa0e16cb1 0xffffffffa0e16f47 : jmpq 0xffffffffa0e16e5c 0xffffffffa0e16f4c : mov $0xffffffffa0e48180,%rdi 0xffffffffa0e16f53 : mov $0xffffffffa0e3bffc,%rdx 0xffffffffa0e16f5a : mov $0xffffffffa0e3be84,%rsi 0xffffffffa0e16f61 : xor %eax,%eax 0xffffffffa0e16f63 : movl $0x40000,0x3122b(%rip) # 0xffffffffa0e48198 <__msg_data.69174+24> 0xffffffffa0e16f6d : callq 0xffffffffa09dfeb0 0xffffffffa0e16f72 : mov $0xffffffffa0e48180,%rdi 0xffffffffa0e16f79 : callq 0xffffffffa09cfd20 0xffffffffa0e16f7e : mov $0xffffffffa0e48200,%rdi 0xffffffffa0e16f85 : mov $0xffffffffa0e38188,%rdx 0xffffffffa0e16f8c : mov $0xffffffffa0e3be84,%rsi 0xffffffffa0e16f93 : xor %eax,%eax 0xffffffffa0e16f95 : movl $0x40000,0x31279(%rip) # 0xffffffffa0e48218 <__msg_data.70031+24> 0xffffffffa0e16f9f : callq 0xffffffffa09dfeb0 0xffffffffa0e16fa4 : mov $0xffffffffa0e48200,%rdi 0xffffffffa0e16fab : callq 0xffffffffa09cfd20 == schedule == crash> dis schedule | sed -e 's/^/ /' 0xffffffff8145efb0 : push %rbp 0xffffffff8145efb1 : mov %gs:0xb600,%rdx 0xffffffff8145efba : mov (%rdx),%rax 0xffffffff8145efbd : mov %rsp,%rbp 0xffffffff8145efc0 : test %rax,%rax 0xffffffff8145efc3 : je 0xffffffff8145efe5 0xffffffff8145efc5 : mov 0x810(%rdx),%rdi 0xffffffff8145efcc : test %rdi,%rdi 0xffffffff8145efcf : je 0xffffffff8145efe5 0xffffffff8145efd1 : lea 0x8(%rdi),%rax 0xffffffff8145efd5 : cmp %rax,0x8(%rdi) 0xffffffff8145efd9 : je 0xffffffff8145eff0 0xffffffff8145efdb : mov $0x1,%esi 0xffffffff8145efe0 : callq 0xffffffff81227ff0 0xffffffff8145efe5 : leaveq 0xffffffff8145efe6 : jmpq 0xffffffff8145e750 <__schedule> 0xffffffff8145efeb : nopl 0x0(%rax,%rax,1) 0xffffffff8145eff0 : lea 0x18(%rdi),%rax 0xffffffff8145eff4 : cmp %rax,0x18(%rdi) 0xffffffff8145eff8 : jne 0xffffffff8145efdb 0xffffffff8145effa : jmp 0xffffffff8145efe5 0xffffffff8145effc : nopl 0x0(%rax) == __schedule == crash> dis __schedule | sed -e 's/^/ /' 0xffffffff8145e750 <__schedule>: push %rbp 0xffffffff8145e751 <__schedule+1>: mov $0x11800,%rax 0xffffffff8145e758 <__schedule+8>: mov %gs:0xb608,%rdx 0xffffffff8145e761 <__schedule+17>: mov %rsp,%rbp 0xffffffff8145e764 <__schedule+20>: push %r15 0xffffffff8145e766 <__schedule+22>: mov %rdx,%rcx 0xffffffff8145e769 <__schedule+25>: sub $0x1fc8,%rcx 0xffffffff8145e770 <__schedule+32>: push %r14 0xffffffff8145e772 <__schedule+34>: push %r13 0xffffffff8145e774 <__schedule+36>: push %r12 0xffffffff8145e776 <__schedule+38>: push %rbx 0xffffffff8145e777 <__schedule+39>: sub $0x108,%rsp 0xffffffff8145e77e <__schedule+46>: mov %rax,-0xf8(%rbp) 0xffffffff8145e785 <__schedule+53>: mov %rdx,-0x100(%rbp) 0xffffffff8145e78c <__schedule+60>: mov %rdx,-0x108(%rbp) 0xffffffff8145e793 <__schedule+67>: mov %rax,-0x110(%rbp) 0xffffffff8145e79a <__schedule+74>: mov %rax,-0x118(%rbp) 0xffffffff8145e7a1 <__schedule+81>: mov %rax,-0x120(%rbp) 0xffffffff8145e7a8 <__schedule+88>: mov %rax,-0x128(%rbp) 0xffffffff8145e7af <__schedule+95>: mov %rcx,-0x130(%rbp) 0xffffffff8145e7b6 <__schedule+102>: nopw %cs:0x0(%rax,%rax,1) 0xffffffff8145e7c0 <__schedule+112>: mov %gs:0xcbb0,%ebx 0xffffffff8145e7c8 <__schedule+120>: movslq %ebx,%rax 0xffffffff8145e7cb <__schedule+123>: mov -0xf8(%rbp),%r13 0xffffffff8145e7d2 <__schedule+130>: mov %ebx,%edi 0xffffffff8145e7d4 <__schedule+132>: add -0x7e436c00(,%rax,8),%r13 0xffffffff8145e7dc <__schedule+140>: callq 0xffffffff810cbfe0 0xffffffff8145e7e1 <__schedule+145>: mov -0x100(%rbp),%rdi 0xffffffff8145e7e8 <__schedule+152>: mov 0x888(%r13),%rsi 0xffffffff8145e7ef <__schedule+159>: mov %rsi,-0xf0(%rbp) 0xffffffff8145e7f6 <__schedule+166>: testl $0xefffffff,-0x1fbc(%rdi) 0xffffffff8145e800 <__schedule+176>: je 0xffffffff8145e810 <__schedule+192> 0xffffffff8145e802 <__schedule+178>: mov 0x484(%rsi),%ecx 0xffffffff8145e808 <__schedule+184>: test %ecx,%ecx 0xffffffff8145e80a <__schedule+186>: je 0xffffffff8145edc0 0xffffffff8145e810 <__schedule+192>: cmpl $0x2,0x7759f9(%rip) # 0xffffffff81bd4210 0xffffffff8145e817 <__schedule+199>: mov 0x8(%rbp),%rsi 0xffffffff8145e81b <__schedule+203>: je 0xffffffff8145ed90 0xffffffff8145e821 <__schedule+209>: mov $0x11800,%rax 0xffffffff8145e828 <__schedule+216>: add %gs:0xcba8,%rax 0xffffffff8145e831 <__schedule+225>: addl $0x1,0xa18(%rax) 0xffffffff8145e838 <__schedule+232>: testb $0x1,0x7750a2(%rip) # 0xffffffff81bd38e1 0xffffffff8145e83f <__schedule+239>: je 0xffffffff8145e84f <__schedule+255> 0xffffffff8145e841 <__schedule+241>: cmpq $0x0,0x9c0(%r13) 0xffffffff8145e849 <__schedule+249>: jne 0xffffffff8145edd0 0xffffffff8145e84f <__schedule+255>: mov %r13,%rdi 0xffffffff8145e852 <__schedule+258>: callq 0xffffffff81460ec0 <_raw_spin_lock_irq> 0xffffffff8145e857 <__schedule+263>: mov -0xf0(%rbp),%rdx 0xffffffff8145e85e <__schedule+270>: mov -0xf0(%rbp),%r12 0xffffffff8145e865 <__schedule+277>: mov (%rdx),%rax 0xffffffff8145e868 <__schedule+280>: add $0x5b0,%r12 0xffffffff8145e86f <__schedule+287>: test %rax,%rax 0xffffffff8145e872 <__schedule+290>: je 0xffffffff8145e8cd <__schedule+381> 0xffffffff8145e874 <__schedule+292>: mov -0x108(%rbp),%rcx 0xffffffff8145e87b <__schedule+299>: testb $0x10,-0x1fb9(%rcx) 0xffffffff8145e882 <__schedule+306>: jne 0xffffffff8145e8cd <__schedule+381> 0xffffffff8145e884 <__schedule+308>: mov %rdx,%rsi 0xffffffff8145e887 <__schedule+311>: mov (%rdx),%rdx 0xffffffff8145e88a <__schedule+314>: test $0x81,%dl 0xffffffff8145e88d <__schedule+317>: jne 0xffffffff8145ee05 0xffffffff8145e893 <__schedule+323>: mov -0xf0(%rbp),%rsi 0xffffffff8145e89a <__schedule+330>: mov %r13,%rdi 0xffffffff8145e89d <__schedule+333>: mov $0x1,%edx 0xffffffff8145e8a2 <__schedule+338>: callq 0xffffffff8104c8d0 0xffffffff8145e8a7 <__schedule+343>: mov -0xf0(%rbp),%rdi 0xffffffff8145e8ae <__schedule+350>: testb $0x20,0x14(%rdi) 0xffffffff8145e8b2 <__schedule+354>: movl $0x0,0x2c(%rdi) 0xffffffff8145e8b9 <__schedule+361>: jne 0xffffffff8145ede1 0xffffffff8145e8bf <__schedule+367>: mov -0xf0(%rbp),%r12 0xffffffff8145e8c6 <__schedule+374>: add $0x5a8,%r12 0xffffffff8145e8cd <__schedule+381>: mov -0xf0(%rbp),%rdx 0xffffffff8145e8d4 <__schedule+388>: mov 0x40(%rdx),%rax 0xffffffff8145e8d8 <__schedule+392>: mov 0x48(%rax),%rax 0xffffffff8145e8dc <__schedule+396>: test %rax,%rax 0xffffffff8145e8df <__schedule+399>: je 0xffffffff8145e8e9 <__schedule+409> 0xffffffff8145e8e1 <__schedule+401>: mov %rdx,%rsi 0xffffffff8145e8e4 <__schedule+404>: mov %r13,%rdi 0xffffffff8145e8e7 <__schedule+407>: callq *%rax 0xffffffff8145e8e9 <__schedule+409>: cmpq $0x0,0x8(%r13) 0xffffffff8145e8ee <__schedule+414>: je 0xffffffff8145eed0 0xffffffff8145e8f4 <__schedule+420>: mov -0xf0(%rbp),%rcx 0xffffffff8145e8fb <__schedule+427>: mov 0x2c(%rcx),%edx 0xffffffff8145e8fe <__schedule+430>: test %edx,%edx 0xffffffff8145e900 <__schedule+432>: jne 0xffffffff8145ed00 0xffffffff8145e906 <__schedule+438>: mov 0x4c(%r13),%eax 0xffffffff8145e90a <__schedule+442>: test %eax,%eax 0xffffffff8145e90c <__schedule+444>: js 0xffffffff8145ed00 0xffffffff8145e912 <__schedule+450>: mov -0xf0(%rbp),%rbx 0xffffffff8145e919 <__schedule+457>: mov %r13,%rdi 0xffffffff8145e91c <__schedule+460>: mov 0x40(%rbx),%rax 0xffffffff8145e920 <__schedule+464>: mov %rbx,%rsi 0xffffffff8145e923 <__schedule+467>: callq *0x38(%rax) 0xffffffff8145e926 <__schedule+470>: mov 0x88(%r13),%rsi 0xffffffff8145e92d <__schedule+477>: cmp %rsi,0x8(%r13) 0xffffffff8145e931 <__schedule+481>: jne 0xffffffff8145ee3c 0xffffffff8145e937 <__schedule+487>: mov %r13,%rdi 0xffffffff8145e93a <__schedule+490>: callq 0xffffffff810598e0 0xffffffff8145e93f <__schedule+495>: test %rax,%rax 0xffffffff8145e942 <__schedule+498>: mov %rax,-0xe8(%rbp) 0xffffffff8145e949 <__schedule+505>: je 0xffffffff8145ee3c 0xffffffff8145e94f <__schedule+511>: mov -0xf0(%rbp),%rdi 0xffffffff8145e956 <__schedule+518>: mov 0x8(%rdi),%rdx 0xffffffff8145e95a <__schedule+522>: lea 0x10(%rdx),%rax 0xffffffff8145e95e <__schedule+526>: lock andb $0xf7,0x10(%rdx) 0xffffffff8145e963 <__schedule+531>: movl $0x0,0x4c(%r13) 0xffffffff8145e96b <__schedule+539>: mov -0xe8(%rbp),%rax 0xffffffff8145e972 <__schedule+546>: cmp %rax,%rdi 0xffffffff8145e975 <__schedule+549>: je 0xffffffff8145eef4 0xffffffff8145e97b <__schedule+555>: addq $0x1,0x68(%r13) 0xffffffff8145e980 <__schedule+560>: mov %rax,0x888(%r13) 0xffffffff8145e987 <__schedule+567>: addq $0x1,(%r12) 0xffffffff8145e98c <__schedule+572>: mov 0x8(%rdi),%rax 0xffffffff8145e990 <__schedule+576>: mov -0x110(%rbp),%rsi 0xffffffff8145e997 <__schedule+583>: mov 0x18(%rax),%eax 0xffffffff8145e99a <__schedule+586>: mov -0x7e436c00(,%rax,8),%rax 0xffffffff8145e9a2 <__schedule+594>: add %rax,%rsi 0xffffffff8145e9a5 <__schedule+597>: mov 0x890(%rsi),%rcx 0xffffffff8145e9ac <__schedule+604>: cmp %rcx,%rdi 0xffffffff8145e9af <__schedule+607>: mov %rcx,%rdx 0xffffffff8145e9b2 <__schedule+610>: je 0xffffffff8145ea3d <__schedule+749> 0xffffffff8145e9b8 <__schedule+616>: mov -0x110(%rbp),%rdx 0xffffffff8145e9bf <__schedule+623>: test %rsi,%rsi 0xffffffff8145e9c2 <__schedule+626>: mov 0x8b0(%rdx,%rax,1),%rax 0xffffffff8145e9ca <__schedule+634>: mov 0x418(%rdi),%rdx 0xffffffff8145e9d1 <__schedule+641>: je 0xffffffff8145e9e4 <__schedule+660> 0xffffffff8145e9d3 <__schedule+643>: add 0xa08(%rsi),%rax 0xffffffff8145e9da <__schedule+650>: sub %rdx,%rax 0xffffffff8145e9dd <__schedule+653>: mov %rax,0xa08(%rsi) 0xffffffff8145e9e4 <__schedule+660>: mov -0xf0(%rbp),%rbx 0xffffffff8145e9eb <__schedule+667>: mov %rcx,%rdx 0xffffffff8145e9ee <__schedule+670>: mov (%rbx),%rax 0xffffffff8145e9f1 <__schedule+673>: test %rax,%rax 0xffffffff8145e9f4 <__schedule+676>: jne 0xffffffff8145ea3d <__schedule+749> 0xffffffff8145e9f6 <__schedule+678>: mov -0xf0(%rbp),%rdi 0xffffffff8145e9fd <__schedule+685>: mov %rcx,%rdx 0xffffffff8145ea00 <__schedule+688>: cmpq $0x0,0x420(%rdi) 0xffffffff8145ea08 <__schedule+696>: jne 0xffffffff8145ea3d <__schedule+749> 0xffffffff8145ea0a <__schedule+698>: mov -0xf0(%rbp),%rcx 0xffffffff8145ea11 <__schedule+705>: mov $0x11800,%rdx 0xffffffff8145ea18 <__schedule+712>: mov 0x8(%rcx),%rax 0xffffffff8145ea1c <__schedule+716>: mov 0x18(%rax),%eax 0xffffffff8145ea1f <__schedule+719>: mov -0x7e436c00(,%rax,8),%rax 0xffffffff8145ea27 <__schedule+727>: mov 0x8b0(%rdx,%rax,1),%rax 0xffffffff8145ea2f <__schedule+735>: mov %rax,0x420(%rcx) 0xffffffff8145ea36 <__schedule+742>: mov 0x890(%rsi),%rdx 0xffffffff8145ea3d <__schedule+749>: cmp %rdx,-0xe8(%rbp) 0xffffffff8145ea44 <__schedule+756>: je 0xffffffff8145eac5 <__schedule+885> 0xffffffff8145ea46 <__schedule+758>: mov -0xe8(%rbp),%rbx 0xffffffff8145ea4d <__schedule+765>: mov -0x120(%rbp),%rdi 0xffffffff8145ea54 <__schedule+772>: mov 0x8(%rbx),%rsi 0xffffffff8145ea58 <__schedule+776>: mov 0x18(%rsi),%eax 0xffffffff8145ea5b <__schedule+779>: mov -0x7e436c00(,%rax,8),%rax 0xffffffff8145ea63 <__schedule+787>: mov 0x8b0(%rdi,%rax,1),%rcx 0xffffffff8145ea6b <__schedule+795>: mov 0x420(%rbx),%rax 0xffffffff8145ea72 <__schedule+802>: xor %edi,%edi 0xffffffff8145ea74 <__schedule+804>: movq $0x0,0x420(%rbx) 0xffffffff8145ea7f <__schedule+815>: mov %rcx,%rdx 0xffffffff8145ea82 <__schedule+818>: mov %rcx,0x418(%rbx) 0xffffffff8145ea89 <__schedule+825>: sub %rax,%rdx 0xffffffff8145ea8c <__schedule+828>: test %rax,%rax 0xffffffff8145ea8f <__schedule+831>: cmovne %rdx,%rdi 0xffffffff8145ea93 <__schedule+835>: addq $0x1,0x408(%rbx) 0xffffffff8145ea9b <__schedule+843>: add %rdi,0x410(%rbx) 0xffffffff8145eaa2 <__schedule+850>: mov 0x18(%rsi),%eax 0xffffffff8145eaa5 <__schedule+853>: mov -0x128(%rbp),%rdx 0xffffffff8145eaac <__schedule+860>: add -0x7e436c00(,%rax,8),%rdx 0xffffffff8145eab4 <__schedule+868>: je 0xffffffff8145eac5 <__schedule+885> 0xffffffff8145eab6 <__schedule+870>: add %rdi,0x9f0(%rdx) 0xffffffff8145eabd <__schedule+877>: addq $0x1,0x9e8(%rdx) 0xffffffff8145eac5 <__schedule+885>: mov 0x9f2f01(%rip),%eax # 0xffffffff81e519cc 0xffffffff8145eacb <__schedule+891>: test %eax,%eax 0xffffffff8145eacd <__schedule+893>: je 0xffffffff8145eb14 <__schedule+964> 0xffffffff8145eacf <__schedule+895>: lea -0xe0(%rbp),%rdx 0xffffffff8145ead6 <__schedule+902>: xor %eax,%eax 0xffffffff8145ead8 <__schedule+904>: mov $0x15,%ecx 0xffffffff8145eadd <__schedule+909>: mov $0x1,%esi 0xffffffff8145eae2 <__schedule+914>: mov %rdx,%rdi 0xffffffff8145eae5 <__schedule+917>: rep stos %rax,%es:(%rdi) 0xffffffff8145eae8 <__schedule+920>: mov $0x3,%edi 0xffffffff8145eaed <__schedule+925>: movq $0x10,-0x58(%rbp) 0xffffffff8145eaf5 <__schedule+933>: movq $0x0,-0x50(%rbp) 0xffffffff8145eafd <__schedule+941>: mov 0x8(%rbp),%rax 0xffffffff8145eb01 <__schedule+945>: mov %rax,-0x60(%rbp) 0xffffffff8145eb05 <__schedule+949>: mov %rbp,%rax 0xffffffff8145eb08 <__schedule+952>: mov %rax,-0xc0(%rbp) 0xffffffff8145eb0f <__schedule+959>: callq 0xffffffff810f30d0 <__perf_sw_event> 0xffffffff8145eb14 <__schedule+964>: mov 0x776d86(%rip),%eax # 0xffffffff81bd58a0 0xffffffff8145eb1a <__schedule+970>: test %eax,%eax 0xffffffff8145eb1c <__schedule+972>: je 0xffffffff8145eb31 <__schedule+993> 0xffffffff8145eb1e <__schedule+974>: mov -0xe8(%rbp),%rsi 0xffffffff8145eb25 <__schedule+981>: mov -0xf0(%rbp),%rdi 0xffffffff8145eb2c <__schedule+988>: callq 0xffffffff810eef70 <__perf_event_task_sched_out> 0xffffffff8145eb31 <__schedule+993>: mov -0xf0(%rbp),%rax 0xffffffff8145eb38 <__schedule+1000>: mov 0x1f0(%rax),%rbx 0xffffffff8145eb3f <__schedule+1007>: test %rbx,%rbx 0xffffffff8145eb42 <__schedule+1010>: je 0xffffffff8145eb61 <__schedule+1041> 0xffffffff8145eb44 <__schedule+1012>: nopl 0x0(%rax) 0xffffffff8145eb48 <__schedule+1016>: mov 0x10(%rbx),%rax 0xffffffff8145eb4c <__schedule+1020>: mov %rbx,%rdi 0xffffffff8145eb4f <__schedule+1023>: mov -0xe8(%rbp),%rsi 0xffffffff8145eb56 <__schedule+1030>: callq *0x8(%rax) 0xffffffff8145eb59 <__schedule+1033>: mov (%rbx),%rbx 0xffffffff8145eb5c <__schedule+1036>: test %rbx,%rbx 0xffffffff8145eb5f <__schedule+1039>: jne 0xffffffff8145eb48 <__schedule+1016> 0xffffffff8145eb61 <__schedule+1041>: mov -0xe8(%rbp),%rdx 0xffffffff8145eb68 <__schedule+1048>: movl $0x1,0x28(%rdx) 0xffffffff8145eb6f <__schedule+1055>: mov 0x760833(%rip),%eax # 0xffffffff81bbf3a8 0xffffffff8145eb75 <__schedule+1061>: test %eax,%eax 0xffffffff8145eb77 <__schedule+1063>: je 0xffffffff8145ebb0 <__schedule+1120> 0xffffffff8145eb79 <__schedule+1065>: mov 0x760840(%rip),%rbx # 0xffffffff81bbf3c0 0xffffffff8145eb80 <__schedule+1072>: test %rbx,%rbx 0xffffffff8145eb83 <__schedule+1075>: je 0xffffffff8145ebb0 <__schedule+1120> 0xffffffff8145eb85 <__schedule+1077>: mov (%rbx),%rax 0xffffffff8145eb88 <__schedule+1080>: nopl 0x0(%rax,%rax,1) 0xffffffff8145eb90 <__schedule+1088>: mov 0x8(%rbx),%rdi 0xffffffff8145eb94 <__schedule+1092>: add $0x10,%rbx 0xffffffff8145eb98 <__schedule+1096>: mov -0xe8(%rbp),%rdx 0xffffffff8145eb9f <__schedule+1103>: mov -0xf0(%rbp),%rsi 0xffffffff8145eba6 <__schedule+1110>: callq *%rax 0xffffffff8145eba8 <__schedule+1112>: mov (%rbx),%rax 0xffffffff8145ebab <__schedule+1115>: test %rax,%rax 0xffffffff8145ebae <__schedule+1118>: jne 0xffffffff8145eb90 <__schedule+1088> 0xffffffff8145ebb0 <__schedule+1120>: mov -0xf0(%rbp),%rsi 0xffffffff8145ebb7 <__schedule+1127>: mov -0xe8(%rbp),%rcx 0xffffffff8145ebbe <__schedule+1134>: mov 0x468(%rsi),%r12 0xffffffff8145ebc5 <__schedule+1141>: mov 0x460(%rcx),%rbx 0xffffffff8145ebcc <__schedule+1148>: mov %rsi,%rdi 0xffffffff8145ebcf <__schedule+1151>: data32 data32 xchg %ax,%ax 0xffffffff8145ebd3 <__schedule+1155>: data32 xchg %ax,%ax 0xffffffff8145ebd6 <__schedule+1158>: test %rbx,%rbx 0xffffffff8145ebd9 <__schedule+1161>: je 0xffffffff8145ed50 0xffffffff8145ebdf <__schedule+1167>: mov %gs:0xcbb0,%r14d 0xffffffff8145ebe8 <__schedule+1176>: cmp %rbx,%r12 0xffffffff8145ebeb <__schedule+1179>: je 0xffffffff8145ee70 0xffffffff8145ebf1 <__schedule+1185>: movl $0x1,%gs:0x11788 0xffffffff8145ebfd <__schedule+1197>: mov %rbx,%gs:0x11780 0xffffffff8145ec06 <__schedule+1206>: mov 0x2c0(%rbx),%rax 0xffffffff8145ec0d <__schedule+1213>: lock bts %r14d,(%rax) 0xffffffff8145ec12 <__schedule+1218>: mov 0x48(%rbx),%rdi 0xffffffff8145ec16 <__schedule+1222>: callq 0xffffffff8103c8a0 <__phys_addr> 0xffffffff8145ec1b <__schedule+1227>: mov %rax,%cr3 0xffffffff8145ec1e <__schedule+1230>: mov 0x2c0(%r12),%rax 0xffffffff8145ec26 <__schedule+1238>: lock btr %r14d,(%rax) 0xffffffff8145ec2b <__schedule+1243>: mov 0x2c8(%rbx),%rdx 0xffffffff8145ec32 <__schedule+1250>: cmp %rdx,0x2c8(%r12) 0xffffffff8145ec3a <__schedule+1258>: jne 0xffffffff8145eedf 0xffffffff8145ec40 <__schedule+1264>: mov -0xf0(%rbp),%rcx 0xffffffff8145ec47 <__schedule+1271>: cmpq $0x0,0x460(%rcx) 0xffffffff8145ec4f <__schedule+1279>: je 0xffffffff8145eda8 0xffffffff8145ec55 <__schedule+1285>: mov -0xe8(%rbp),%rsi 0xffffffff8145ec5c <__schedule+1292>: mov -0xf0(%rbp),%rdi 0xffffffff8145ec63 <__schedule+1299>: pushfq 0xffffffff8145ec64 <__schedule+1300>: push %rbp 0xffffffff8145ec65 <__schedule+1301>: mov %rsi,%rbp 0xffffffff8145ec68 <__schedule+1304>: mov %rsp,0x690(%rdi) 0xffffffff8145ec6f <__schedule+1311>: mov 0x690(%rsi),%rsp 0xffffffff8145ec76 <__schedule+1318>: callq 0xffffffff81001670 <__switch_to> = client_obd = struct client_obd *cli = osc_cli(obj); while (!atomic64_add_unless(cli->cl_lru_left, -1, 0)) { rc = osc_lru_reclaim(cli); 0xffffffffa0e16e40 : mov %r13,%rdi 0xffffffffa0e16e48 : callq 0xffffffffa0e166f0 crash> dis __schedule | grep push 0xffffffff8145e750 <__schedule>: push %rbp 0xffffffff8145e764 <__schedule+20>: push %r15 0xffffffff8145e770 <__schedule+32>: push %r14 0xffffffff8145e772 <__schedule+34>: push %r13 <--- 0xffffffff8145e774 <__schedule+36>: push %r12 0xffffffff8145e776 <__schedule+38>: push %rbx 0xffffffff8145ec63 <__schedule+1299>: pushfq 0xffffffff8145ec64 <__schedule+1300>: push %rbp So it is the fourth value pushed to the stack. crash> struct client_obd ffff880431ea5538 | sed -e 's/^/ /' struct client_obd { cl_sem = { count = 0, wait_lock = { { rlock = { raw_lock = { slock = 0 } } } }, wait_list = { next = 0xffff880431ea5548, prev = 0xffff880431ea5548 } }, cl_target_uuid = { uuid = "accfs1-OST0002_UUID\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000" }, cl_import = 0xffff880376de7800, cl_conn_count = 1, cl_default_mds_easize = 0, cl_max_mds_easize = 48, cl_default_mds_cookiesize = 0, cl_max_mds_cookiesize = 32, cl_sp_me = LUSTRE_SP_CLI, cl_sp_to = LUSTRE_SP_OST, cl_flvr_mgc = { sf_rpc = 0, sf_flags = 0, u_rpc = {}, u_bulk = { hash = { hash_alg = 0 '\000' } } }, cl_dirty = 0, cl_dirty_max = 33554432, cl_dirty_transit = 0, cl_avail_grant = 76988416, cl_lost_grant = 0, cl_reserved_grant = 0, cl_cache_waiters = { next = 0xffff880431ea55e0, prev = 0xffff880431ea55e0 }, cl_next_shrink_grant = 4448659273, cl_grant_shrink_list = { next = 0xffff880431ea55f8, prev = 0xffff880431ea55f8 }, cl_grant_shrink_interval = 1200, cl_chunkbits = 12, cl_chunk = 0, cl_extent_tax = 0, cl_loi_list_lock = { lock = { { rlock = { raw_lock = { slock = 83428601 } } } }, time = 4448368023, task = 0x0, func = 0xffffffffa0e35927 "osc_io_unplug0", line = 2250 }, cl_loi_ready_list = { next = 0xffff880431ea5640, prev = 0xffff880431ea5640 }, cl_loi_hp_ready_list = { next = 0xffff880431ea5650, prev = 0xffff880431ea5650 }, cl_loi_write_list = { next = 0xffff880431ea5660, prev = 0xffff880431ea5660 }, cl_loi_read_list = { next = 0xffff880431ea5670, prev = 0xffff880431ea5670 }, cl_r_in_flight = 0, cl_w_in_flight = 0, cl_pending_w_pages = { counter = 0 }, cl_pending_r_pages = { counter = 0 }, cl_max_pages_per_rpc = 256, cl_max_rpcs_in_flight = 8, cl_read_rpc_hist = { oh_lock = { { rlock = { raw_lock = { slock = 65537 } } } }, oh_buckets = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }, cl_write_rpc_hist = { oh_lock = { { rlock = { raw_lock = { slock = 5374034 } } } }, oh_buckets = {0, 3, 2, 2, 2, 3, 4, 7, 7, 8, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }, cl_read_page_hist = { oh_lock = { { rlock = { raw_lock = { slock = 65537 } } } }, oh_buckets = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }, cl_write_page_hist = { oh_lock = { { rlock = { raw_lock = { slock = 5374034 } } } }, oh_buckets = {28, 7, 2, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }, cl_read_offset_hist = { oh_lock = { { rlock = { raw_lock = { slock = 65537 } } } }, oh_buckets = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }, cl_write_offset_hist = { oh_lock = { { rlock = { raw_lock = { slock = 5374034 } } } }, oh_buckets = {40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} }, cl_cache = 0xffff880155dbc0e0, cl_lru_osc = { next = 0xffff8803653cdd10, prev = 0xffff880155dbc0e8 }, cl_lru_left = 0xffff880155dbc100, cl_lru_busy = { counter = 14 }, cl_lru_shrinkers = { counter = 0 }, cl_lru_in_list = { counter = 16 }, cl_lru_list = { next = 0xffff880431d7ea18, prev = 0xffff8803772e6a18 }, cl_lru_list_lock = { lock = { { rlock = { raw_lock = { slock = 16122102 } } } }, time = 4448368023, task = 0x0, func = 0xffffffffa0e34bf0 "osc_lru_shrink", line = 708 }, cl_unstable_count = { counter = 0 }, cl_destroy_in_flight = { counter = 0 }, cl_destroy_waitq = { lock = { { rlock = { raw_lock = { slock = 0 } } } }, task_list = { next = 0xffff880431ea5d50, prev = 0xffff880431ea5d50 } }, cl_rpc_lock = 0x0, cl_close_lock = 0x0, cl_mgc_mutex = { count = { counter = 1 }, wait_lock = { { rlock = { raw_lock = { slock = 0 } } } }, wait_list = { next = 0xffff880431ea5d78, prev = 0xffff880431ea5d78 }, owner = 0x0 }, cl_mgc_los = 0x0, cl_mgc_configs_dir = 0x0, cl_mgc_refcount = { counter = 0 }, cl_mgc_mgsexp = 0x0, cl_checksum = 1, cl_supp_cksum_types = 6, cl_cksum_type = OBD_CKSUM_ADLER, cl_ar = { ar_rc = 0, ar_force_sync = 0, ar_min_xid = 0 }, cl_qchk_stat = 1, cl_seq = 0x0, cl_resends = { counter = 10 }, cl_writeback_work = 0xffff8803652d7400, cl_lru_work = 0xffff880347956000, cl_quota_hash = {0xffff880380e593c0, 0xffff880360bc3680} } == cl_lru_left == crash> struct atomic64_t 0xffff880155dbc100 struct atomic64_t { counter = 0 } == cl_lru_work == struct ptlrpc_request { rq_type = 4711, rq_status = 0, rq_list = { next = 0xffff880347956008, prev = 0xffff880347956008 }, rq_timed_list = { next = 0x0, prev = 0x0 }, rq_history_list = { next = 0xffff880347956028, prev = 0xffff880347956028 }, rq_exp_list = { next = 0xffff880347956038, prev = 0xffff880347956038 }, rq_ops = 0x0, rq_svc_thread = 0x0, rq_history_seq = 0, rq_nrq = { nr_res_ptrs = {0x0, 0x0}, nr_res_idx = 0, nr_initialized = 0, nr_enqueued = 0, nr_started = 0, nr_finalized = 0, nr_node = { chn_index = 0 }, nr_u = { fifo = { fr_list = { next = 0x0, prev = 0x0 }, fr_sequence = 0 }, crr = { cr_round = 0, cr_sequence = 0 }, orr = { or_range = { or_start = 0, or_end = 0 }, or_round = 0, or_sequence = 0, or_key = { { ok_fid = { f_seq = 0, f_oid = 0, f_ver = 0 }, ok_idx = 0 } }, or_orr_set = 0, or_trr_set = 0, or_logical_set = 0, or_physical_set = 0 }, tbf = { tr_list = { next = 0x0, prev = 0x0 }, tr_sequence = 0 } }, ext = 0x0 }, rq_at_index = 0, rq_lock = { { rlock = { raw_lock = { slock = 7471218 } } } }, rq_intr = 0, rq_replied = 0, rq_err = 0, rq_timedout = 0, rq_resend = 0, rq_restart = 0, rq_replay = 0, rq_no_resend = 1, rq_waiting = 0, rq_receiving_reply = 0, rq_no_delay = 1, rq_net_err = 0, rq_wait_ctx = 0, rq_early = 0, rq_must_unlink = 0, rq_memalloc = 0, rq_packed_final = 0, rq_hp = 0, rq_at_linked = 0, rq_reply_truncate = 0, rq_committed = 0, rq_invalid_rqset = 0, rq_generation_set = 0, rq_no_retry_einprogress = 0, rq_allow_replay = 0, rq_unstable = 0, rq_nr_resend = 0, rq_phase = 3955285507, rq_next_phase = 3955285508, rq_refcount = { counter = 1 }, rq_request_portal = 0, rq_reply_portal = 0, rq_nob_received = 0, rq_reqlen = 0, rq_replen = 0, rq_reqmsg = 0x0, rq_repmsg = 0x0, rq_transno = 0, rq_xid = 1468144665523584, rq_replay_list = { next = 0xffff880347956110, prev = 0xffff880347956110 }, rq_cli_ctx = 0x0, rq_svc_ctx = 0x0, rq_ctx_chain = { next = 0x0, prev = 0x0 }, rq_flvr = { sf_rpc = 0, sf_flags = 0, u_rpc = {}, u_bulk = { hash = { hash_alg = 0 '\000' } } }, rq_sp_from = LUSTRE_SP_CLI, rq_ctx_init = 0, rq_ctx_fini = 0, rq_bulk_read = 0, rq_bulk_write = 0, rq_auth_gss = 0, rq_auth_remote = 0, rq_auth_usr_root = 0, rq_auth_usr_mdt = 0, rq_auth_usr_ost = 0, rq_pack_udesc = 0, rq_pack_bulk = 0, rq_no_reply = 0, rq_pill_init = 0, rq_auth_uid = 0, rq_auth_mapped_uid = 0, rq_user_desc = 0x0, rq_reqbuf = 0x0, rq_repbuf = 0x0, rq_repdata = 0x0, rq_clrbuf = 0x0, rq_reqbuf_len = 0, rq_reqdata_len = 0, rq_repbuf_len = 0, rq_repdata_len = 0, rq_clrbuf_len = 0, rq_clrdata_len = 0, rq_reply_off = 0, rq_req_swab_mask = 0, rq_rep_swab_mask = 0, rq_import_generation = 1, rq_send_state = LUSTRE_IMP_FULL, rq_early_count = 0, rq_req_md_h = { cookie = 0 }, rq_req_cbid = { cbid_fn = 0, cbid_arg = 0x0 }, rq_delay_limit = 0, rq_queued_time = 4448368023, rq_arrival_time = { tv_sec = 0, tv_usec = 0 }, rq_reply_state = 0x0, rq_rqbd = 0x0, rq_reply_md_h = { cookie = 0 }, rq_reply_waitq = { lock = { { rlock = { raw_lock = { slock = 0 } } } }, task_list = { next = 0xffff880347956210, prev = 0xffff880347956210 } }, rq_reply_cbid = { cbid_fn = 0, cbid_arg = 0x0 }, rq_self = 0, rq_peer = { nid = 0, pid = 0 }, rq_export = 0x0, rq_import = 0xffff880376de7800, rq_replay_cb = 0, rq_commit_cb = 0, rq_cb_data = 0x0, rq_bulk = 0x0, rq_sent = 1400134007, rq_real_sent = 0, rq_deadline = 1400134027, rq_reply_deadline = 1400134027, rq_bulk_deadline = 0, rq_timeout = 20, rq_set_waitq = { lock = { { rlock = { raw_lock = { slock = 0 } } } }, task_list = { next = 0xffff8803479562b0, prev = 0xffff8803479562b0 } }, rq_set_chain = { next = 0xffff8803479562c0, prev = 0xffff8803479562c0 }, rq_set = 0x0, rq_interpret_reply = 0xffffffffa0c8c9f0 , rq_async_args = { pointer_arg = {0xffffffffa0e171e0, 0xffff880431ea5538, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, space = {18446744072113713632, 18446612150331528504, 0, 0, 0, 0, 0} }, rq_pool = 0x0, rq_session = { lc_tags = 0, lc_state = 0, lc_thread = 0x0, lc_value = 0x0, lc_remember = { next = 0x0, prev = 0x0 }, lc_version = 0, lc_cookie = 0 }, rq_pill = { rc_req = 0x0, rc_fmt = 0xffffffffa0d51d4c, rc_loc = RCL_CLIENT, rc_area = {{0, 0, 0, 0, 0, 0, 0, 0, 0}, {0, 0, 0, 0, 0, 0, 0, 0, 0}} } } == osc_lru_shrink == osc_lru_shrink is probably what we are looking for. long osc_lru_shrink(const struct lu_env *env, struct client_obd *cli, long target, bool force) { struct cl_io *io; struct cl_object *clobj = NULL; struct cl_page **pvec; struct osc_page *opg; long maxscan = 0; long count = 0; int index = 0; int rc = 0; ENTRY; LASSERT(atomic64_read(&cli->cl_lru_in_list) >= 0); Currently at 0. if (atomic64_read(&cli->cl_lru_in_list) == 0 || target <= 0) RETURN(0); at least half of this is true. target, of course, depends on the caller. crash> struct client_obd.cl_lru_in_list ffff880431ea5538 cl_lru_in_list = { counter = 16 } if (!force) { if (atomic_read(&cli->cl_lru_shrinkers) > 0) RETURN(-EBUSY); if (atomic_inc_return(&cli->cl_lru_shrinkers) > 1) { atomic_dec(&cli->cl_lru_shrinkers); RETURN(-EBUSY); } } else { atomic_inc(&cli->cl_lru_shrinkers); } crash> struct client_obd.cl_lru_shrinkers ffff880431ea5538 cl_lru_shrinkers = { counter = 0 } So there are currently no processes in osc_lru_shrink which would prevent a shrink. maxscan = min(target << 1, atomic64_read(&cli->cl_lru_in_list)); while (!list_empty(&cli->cl_lru_list)) { struct cl_page *page; bool will_free = false; if (--maxscan < 0) break; opg = list_entry(cli->cl_lru_list.next, struct osc_page, ops_lru); page = opg->ops_cl.cpl_page; if (cl_page_in_use_noref(page)) { list_move_tail(&opg->ops_lru, &cli->cl_lru_list); continue; } == osc_cache_too_much == static int osc_cache_too_much(struct client_obd *cli) { struct cl_client_cache *cache = cli->cl_cache; long pages = atomic64_read(&cli->cl_lru_in_list); unsigned long budget; budget = cache->ccc_lru_max / atomic_read(&cache->ccc_users); /* if it's going to run out LRU slots, we should free some, but not * too much to maintain faireness among OSCs. */ if (atomic64_read(cli->cl_lru_left) < cache->ccc_lru_max >> 4) { if (pages >= budget) return lru_shrink_max; else if (pages >= budget / 2) return lru_shrink_min; } else if (pages >= budget * 2) return lru_shrink_min; return 0; } crash> struct client_obd.cl_lru_in_list ffff880431ea5538 cl_lru_in_list = { counter = 16 } crash> struct cl_client_cache 0xffff880155dbc0e0 struct cl_client_cache { ccc_users = { counter = 4 }, ccc_lru_max = 32, ccc_lru_left = { counter = 0 }, crash> px ( 32 >> 4 ) $12 = 0x2 static const int lru_shrink_max = 8 << (20 - PAGE_CACHE_SHIFT); /* 8M */ So basically: if (cl_lru_left=0 < 2 ) if pages=16 >= budget=8 return 2048 === the page list === crash> struct client_obd.cl_lru_in_list ffff880431ea5538 cl_lru_in_list = { counter = 16 } crash> struct client_obd.cl_lru_list ffff880431ea5538 cl_lru_list = { next = 0xffff880431d7ea18, prev = 0xffff8803772e6a18 } crash> list 0xffff880431d7ea18 ffff880431d7ea18 ffff880386096218 ffff880376c65e18 ffff880362cf4218 ffff88037f49fa18 ffff8803811ef218 ffff8803811ef618 ffff880380d12618 ffff88037856b218 ffff8803820d9a18 ffff880377565e18 ffff88036f5c8618 ffff88036f5c8a18 ffff88036f5c8218 ffff8803772e6618 ffff8803772e6a18 ffff880431ea5d00 crash> struct -o client_obd | grep cl_lru_list [1992] cfs_list_t cl_lru_list; [2008] client_obd_lock_t cl_lru_list_lock; crash> px ( 0xffff880431ea5538 + 1992 ) $23 = 0xffff880431ea5d00 crash> struct -o osc_page | grep ops_lru [184] struct list_head ops_lru; px ( 0xffff880431d7ea18 - 184 ) = 0xffff880431d7e960 px ( 0xffff880386096218 - 184 ) = 0xffff880386096160 px ( 0xffff880376c65e18 - 184 ) = 0xffff880376c65d60 px ( 0xffff880362cf4218 - 184 ) = 0xffff880362cf4160 px ( 0xffff88037f49fa18 - 184 ) = 0xffff88037f49f960 px ( 0xffff8803811ef218 - 184 ) = 0xffff8803811ef160 px ( 0xffff8803811ef618 - 184 ) = 0xffff8803811ef560 px ( 0xffff880380d12618 - 184 ) = 0xffff880380d12560 px ( 0xffff88037856b218 - 184 ) = 0xffff88037856b160 px ( 0xffff8803820d9a18 - 184 ) = 0xffff8803820d9960 px ( 0xffff880377565e18 - 184 ) = 0xffff880377565d60 px ( 0xffff88036f5c8618 - 184 ) = 0xffff88036f5c8560 px ( 0xffff88036f5c8a18 - 184 ) = 0xffff88036f5c8960 px ( 0xffff88036f5c8218 - 184 ) = 0xffff88036f5c8160 px ( 0xffff8803772e6618 - 184 ) = 0xffff8803772e6560 px ( 0xffff8803772e6a18 - 184 ) = 0xffff8803772e6960 === the pages === crash> struct osc_page.ops_cl 0xffff880431d7e960 ops_cl = { cpl_page = 0xffff880431d7e800, cpl_index = 15, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff880431d7e810, prev = 0xffff880431d7e950 } } crash> struct osc_page.ops_cl 0xffff880386096160 ops_cl = { cpl_page = 0xffff880386096000, cpl_index = 14, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff880386096010, prev = 0xffff880386096150 } } crash> struct osc_page.ops_cl 0xffff880376c65d60 ops_cl = { cpl_page = 0xffff880376c65c00, cpl_index = 13, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff880376c65c10, prev = 0xffff880376c65d50 } } crash> struct osc_page.ops_cl 0xffff880362cf4160 ops_cl = { cpl_page = 0xffff880362cf4000, cpl_index = 12, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff880362cf4010, prev = 0xffff880362cf4150 } } crash> struct osc_page.ops_cl 0xffff88037f49f960 ops_cl = { cpl_page = 0xffff88037f49f800, cpl_index = 11, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff88037f49f810, prev = 0xffff88037f49f950 } } crash> struct osc_page.ops_cl 0xffff8803811ef160 ops_cl = { cpl_page = 0xffff8803811ef000, cpl_index = 10, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff8803811ef010, prev = 0xffff8803811ef150 } } crash> struct osc_page.ops_cl 0xffff8803811ef560 ops_cl = { cpl_page = 0xffff8803811ef400, cpl_index = 9, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff8803811ef410, prev = 0xffff8803811ef550 } } crash> struct osc_page.ops_cl 0xffff880380d12560 ops_cl = { cpl_page = 0xffff880380d12400, cpl_index = 8, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff880380d12410, prev = 0xffff880380d12550 } } crash> struct osc_page.ops_cl 0xffff88037856b160 ops_cl = { cpl_page = 0xffff88037856b000, cpl_index = 7, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff88037856b010, prev = 0xffff88037856b150 } } crash> struct osc_page.ops_cl 0xffff8803820d9960 ops_cl = { cpl_page = 0xffff8803820d9800, cpl_index = 6, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff8803820d9810, prev = 0xffff8803820d9950 } } crash> struct osc_page.ops_cl 0xffff880377565d60 ops_cl = { cpl_page = 0xffff880377565c00, cpl_index = 5, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff880377565c10, prev = 0xffff880377565d50 } } crash> struct osc_page.ops_cl 0xffff88036f5c8560 ops_cl = { cpl_page = 0xffff88036f5c8400, cpl_index = 4, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff88036f5c8410, prev = 0xffff88036f5c8550 } } crash> struct osc_page.ops_cl 0xffff88036f5c8960 ops_cl = { cpl_page = 0xffff88036f5c8800, cpl_index = 3, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff88036f5c8810, prev = 0xffff88036f5c8950 } } crash> struct osc_page.ops_cl 0xffff88036f5c8160 ops_cl = { cpl_page = 0xffff88036f5c8000, cpl_index = 2, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff88036f5c8010, prev = 0xffff88036f5c8150 } } crash> struct osc_page.ops_cl 0xffff8803772e6560 ops_cl = { cpl_page = 0xffff8803772e6400, cpl_index = 1, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff8803772e6410, prev = 0xffff8803772e6550 } } crash> struct osc_page.ops_cl 0xffff8803772e6960 ops_cl = { cpl_page = 0xffff8803772e6800, cpl_index = 0, cpl_obj = 0xffff880352ecfc48, cpl_ops = 0xffffffffa0e34c60, cpl_linkage = { next = 0xffff8803772e6810, prev = 0xffff8803772e6950 } } === cl_pages === crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff880431d7e800 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff880386096000 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff880376c65c00 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff880362cf4000 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff88037f49f800 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff8803811ef000 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff8803811ef400 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff880380d12400 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff88037856b000 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff8803820d9800 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff880377565c00 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff88036f5c8400 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff88036f5c8800 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff88036f5c8000 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff8803772e6400 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED crash> struct cl_page.cp_type,cp_ref,cp_state 0xffff8803772e6800 cp_type = CPT_CACHEABLE cp_ref = { counter = 1 } cp_state = CPS_CACHED static inline int __page_in_use(const struct cl_page *page, int refc) { if (page->cp_type == CPT_CACHEABLE) ++refc; LASSERT(atomic_read(&page->cp_ref) > 0); return (atomic_read(&page->cp_ref) > refc); } #define cl_page_in_use(pg) __page_in_use(pg, 1) #define cl_page_in_use_noref(pg) __page_in_use(pg, 0) static inline struct page *cl_page_vmpage(struct cl_page *page) { LASSERT(page->cp_vmpage != NULL); return page->cp_vmpage; } == osc_lru_reclaim == 00000008:00000001:5.0:1400133972.928496:0:22025:0:(osc_page.c:878:osc_lru_reserve()) Process entered 00000008:00000020:5.0:1400133972.928506:0:22025:0:(osc_page.c:833:osc_lru_reclaim()) accfs1-OST0002-osc-ffff88022017e000: cli ffff880431ea5538 no free slots, pages: 0, busy: 30. 00000008:00000020:5.0:1400133972.928513:0:22025:0:(osc_page.c:867:osc_lru_reclaim()) accfs1-OST0000-osc- ffff88022017e000: cli ffff880430d09538 freed 0 pages. This is the last we heard from the hung process... Now only 14 are busy: crash> struct client_obd.cl_lru_busy ffff880431ea5538 cl_lru_busy = { counter = 14 } So somehow, this changed without a wakeup. There should be cl_lru_max=32 entries. 14 are in cl_lru_busy, 16 are in cl_lru_in_list. The other two from the pool are here: crash> struct client_obd ffff8804316575f8 cl_target_uuid = { uuid = "accfs1-OST0001_UUID\000\000\000\000\000\000\000\000\000\000\000\000\ 000\000\000\000\000\000\000\000" }, cl_lru_left = 0xffff880155dbc100, cl_lru_busy = { counter = 0 }, cl_lru_shrinkers = { counter = 0 }, cl_lru_in_list = { counter = 2 }, == osc_lru_add_batch == This is where entries are moved from busy to "in list". void osc_lru_add_batch(struct client_obd *cli, struct list_head *plist) { if (npages > 0) { client_obd_list_lock(&cli->cl_lru_list_lock); list_splice_tail(&lru, &cli->cl_lru_list); atomic64_sub(npages, &cli->cl_lru_busy); atomic64_add(npages, &cli->cl_lru_in_list); client_obd_list_unlock(&cli->cl_lru_list_lock); /* XXX: May set force to be true for better performance */ if (osc_cache_too_much(cli)) (void)ptlrpcd_queue_work(cli->cl_lru_work); } This work should be in progress... == ptlrpcd_queue_work == int ptlrpcd_queue_work(void *handler) { struct ptlrpc_request *req = handler; /* * Check if the req is already being queued. * * Here comes a trick: it lacks a way of checking if a req is being * processed reliably in ptlrpc. Here I have to use refcount of req * for this purpose. This is okay because the caller should use this * req as opaque data. - Jinshan */ LASSERT(atomic_read(&req->rq_refcount) > 0); if (atomic_inc_return(&req->rq_refcount) == 2) ptlrpcd_add_work_req(req); return 0; } static void ptlrpcd_add_work_req(struct ptlrpc_request *req) { /* re-initialize the req */ req->rq_timeout = obd_timeout; req->rq_sent = cfs_time_current_sec(); req->rq_deadline = req->rq_sent + req->rq_timeout; req->rq_reply_deadline = req->rq_deadline; req->rq_phase = RQ_PHASE_INTERPRET; req->rq_next_phase = RQ_PHASE_COMPLETE; req->rq_xid = ptlrpc_next_xid(); req->rq_import_generation = req->rq_import->imp_generation; ptlrpcd_add_req(req, PDL_POLICY_ROUND, -1); } /** * Requests that are added to the ptlrpcd queue are sent via * ptlrpcd_check->ptlrpc_check_set(). */ void ptlrpcd_add_req(struct ptlrpc_request *req, pdl_policy_t policy, int idx) { struct ptlrpcd_ctl *pc; [ ... ] pc = ptlrpcd_select_pc(req, policy, idx); DEBUG_REQ(D_INFO, req, "add req [%p] to pc [%s:%d]", req, pc->pc_name, pc->pc_index); ptlrpc_set_add_new_req(pc, req); } ptlrpcd_select_pc(struct ptlrpc_request *req, pdl_policy_t policy, int index) { [ ... ] return &ptlrpcds->pd_threads[idx]; } void ptlrpc_set_add_new_req(struct ptlrpcd_ctl *pc, struct ptlrpc_request *req) { struct ptlrpc_request_set *set = pc->pc_set; int count, i; [ ... ] cfs_list_add_tail(&req->rq_set_chain, &set->set_new_requests); [ ... ] for (i = 0; i < pc->pc_npartners; i++) wake_up(&pc->pc_partners[i]->pc_set->set_waitq); } } == cl_lru_work == handler = ptlrpcd_alloc_work(cli->cl_import, lru_queue_work, cli); if (IS_ERR(handler)) GOTO(out_ptlrpcd_work, rc = PTR_ERR(handler)); cli->cl_lru_work = handler; == lru_queue_work == int lru_queue_work(const struct lu_env *env, void *data) { struct client_obd *cli = data; CDEBUG(D_CACHE, "Run LRU work for client obd %p.\n", cli); if (osc_cache_too_much(cli)) osc_lru_shrink(env, cli, lru_shrink_max, true); RETURN(0); }