[91650.837134] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91650.838307] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91650.839778] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91650.841019] ? finish_wait+0x80/0x80 [91650.841817] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91650.842714] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91650.843686] kthread+0x134/0x150 [91650.844325] ? set_kthread_struct+0x50/0x50 [91650.845117] ret_from_fork+0x1f/0x40 [91650.845836] CPU: 17 PID: 527233 Comm: ll_ost_io04_018 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91650.849748] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91650.851678] Call Trace: [91650.852269] dump_stack+0x41/0x60 [91650.852995] warn_alloc.cold.127+0x7b/0x108 [91650.854012] __alloc_pages_slowpath+0xcb2/0xcd0 [91650.855292] ? __alloc_pages_nodemask+0x166/0x330 [91650.856204] __alloc_pages_nodemask+0x2e2/0x330 [91650.857079] kmalloc_order+0x28/0x90 [91650.857818] kmalloc_order_trace+0x1d/0xb0 [91650.858652] __kmalloc+0x203/0x250 [91650.859396] ? __wake_up_common+0x7a/0x190 [91650.860181] virtqueue_add+0x493/0xc70 [91650.860949] ? finish_wait+0x80/0x80 [91650.861685] virtqueue_add_sgs+0x80/0xa0 [91650.862472] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91650.863474] ? scsi_alloc_sgtables+0x84/0x1c0 [91650.864372] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91650.865312] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91650.866395] scsi_queue_rq+0x512/0xb10 [91650.867145] __blk_mq_try_issue_directly+0x163/0x200 [91650.868026] blk_mq_request_issue_directly+0x4e/0xb0 [91650.869093] blk_mq_try_issue_list_directly+0x62/0x100 [91650.870208] blk_mq_sched_insert_requests+0xa4/0xf0 [91650.871377] blk_mq_flush_plug_list+0x135/0x220 [91650.872440] blk_flush_plug_list+0xd7/0x100 [91650.873515] blk_finish_plug+0x25/0x36 [91650.874510] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91650.875927] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91650.877312] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91650.878452] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91650.879459] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91650.880402] ofd_commitrw+0x5b4/0xd20 [ofd] [91650.881468] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91650.882597] obd_commitrw+0x1b6/0x370 [ptlrpc] [91650.883641] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91650.884669] ? flush_work+0x42/0x1d0 [91650.885429] ? internal_add_timer+0x42/0x70 [91650.886396] ? _cond_resched+0x15/0x30 [91650.887266] ? mutex_lock+0xe/0x30 [91650.888180] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91650.889438] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91650.890594] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91650.891704] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91650.892721] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91650.893742] kthread+0x134/0x150 [91650.894389] ? set_kthread_struct+0x50/0x50 [91650.895156] ret_from_fork+0x1f/0x40 [91650.895936] CPU: 18 PID: 528064 Comm: ll_ost_io04_076 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91650.898909] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91650.901083] Call Trace: [91650.901866] dump_stack+0x41/0x60 [91650.902817] warn_alloc.cold.127+0x7b/0x108 [91650.903952] __alloc_pages_slowpath+0xcb2/0xcd0 [91650.905127] __alloc_pages_nodemask+0x2e2/0x330 [91650.906309] kmalloc_order+0x28/0x90 [91650.907295] kmalloc_order_trace+0x1d/0xb0 [91650.908417] __kmalloc+0x203/0x250 [91650.909402] virtqueue_add+0x493/0xc70 [91650.910426] ? finish_wait+0x80/0x80 [91650.911416] virtqueue_add_sgs+0x80/0xa0 [91650.912485] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91650.913904] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91650.915221] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91650.916674] scsi_queue_rq+0x512/0xb10 [91650.917736] __blk_mq_try_issue_directly+0x163/0x200 [91650.919034] blk_mq_request_issue_directly+0x4e/0xb0 [91650.920339] blk_mq_try_issue_list_directly+0x62/0x100 [91650.921621] blk_mq_sched_insert_requests+0xa4/0xf0 [91650.922878] blk_mq_flush_plug_list+0x135/0x220 [91650.924046] blk_flush_plug_list+0xd7/0x100 [91650.925162] blk_finish_plug+0x25/0x36 [91650.926188] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91650.927626] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91650.929019] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91650.930569] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91650.932085] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91650.933401] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91650.934627] ofd_commitrw+0x5b4/0xd20 [ofd] [91650.935808] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91650.937144] obd_commitrw+0x1b6/0x370 [ptlrpc] [91650.938427] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91650.939723] ? flush_work+0x42/0x1d0 [91650.940692] ? internal_add_timer+0x42/0x70 [91650.941764] ? _cond_resched+0x15/0x30 [91650.942745] ? mutex_lock+0xe/0x30 [91650.943709] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91650.945141] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91650.946516] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91650.947588] ? lu_context_exit+0x2f/0x70 [obdclass] [91650.948521] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91650.949416] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91650.950388] kthread+0x134/0x150 [91650.951114] ? set_kthread_struct+0x50/0x50 [91650.951928] ret_from_fork+0x1f/0x40 [91650.952672] CPU: 12 PID: 527035 Comm: ll_ost_io03_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91650.955000] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91650.956778] Call Trace: [91650.957348] dump_stack+0x41/0x60 [91650.958090] warn_alloc.cold.127+0x7b/0x108 [91650.958901] __alloc_pages_slowpath+0xcb2/0xcd0 [91650.959885] ? __alloc_pages_nodemask+0x166/0x330 [91650.960794] __alloc_pages_nodemask+0x2e2/0x330 [91650.961761] kmalloc_order+0x28/0x90 [91650.962651] kmalloc_order_trace+0x1d/0xb0 [91650.963459] __kmalloc+0x203/0x250 [91650.964207] virtqueue_add+0x493/0xc70 [91650.965159] ? finish_wait+0x80/0x80 [91650.965900] virtqueue_add_sgs+0x80/0xa0 [91650.966732] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91650.968004] ? scsi_alloc_sgtables+0x84/0x1c0 [91650.968968] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91650.970170] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91650.971533] scsi_queue_rq+0x512/0xb10 [91650.972468] __blk_mq_try_issue_directly+0x163/0x200 [91650.973532] blk_mq_request_issue_directly+0x4e/0xb0 [91650.974666] blk_mq_try_issue_list_directly+0x62/0x100 [91650.975694] blk_mq_sched_insert_requests+0xa4/0xf0 [91650.976925] blk_mq_flush_plug_list+0x135/0x220 [91650.977851] blk_flush_plug_list+0xd7/0x100 [91650.978943] blk_finish_plug+0x25/0x36 [91650.979918] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91650.981378] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91650.982737] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91650.984231] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91650.985461] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91650.986691] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91650.987697] ofd_commitrw+0x5b4/0xd20 [ofd] [91650.988509] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91650.989484] obd_commitrw+0x1b6/0x370 [ptlrpc] [91650.990469] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91650.991578] ? flush_work+0x42/0x1d0 [91650.992597] ? internal_add_timer+0x42/0x70 [91650.993687] ? _cond_resched+0x15/0x30 [91650.994687] ? mutex_lock+0xe/0x30 [91650.995549] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91650.996758] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91650.997977] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91650.999420] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91651.000565] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91651.001945] kthread+0x134/0x150 [91651.002717] ? set_kthread_struct+0x50/0x50 [91651.003586] ret_from_fork+0x1f/0x40 [91655.382147] warn_alloc: 315 callbacks suppressed [91655.382151] ll_ost_io04_043: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91655.386356] CPU: 17 PID: 527680 Comm: ll_ost_io04_043 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.387692] ll_ost_io04_055: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [91655.389228] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.389232] Call Trace: [91655.391882] / [91655.394016] dump_stack+0x41/0x60 [91655.394918] ,mems_allowed=0 [91655.395658] warn_alloc.cold.127+0x7b/0x108 [91655.396212] ll_ost_io00_033: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91655.397433] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.402580] __alloc_pages_nodemask+0x2e2/0x330 [91655.403469] kmalloc_order+0x28/0x90 [91655.404186] kmalloc_order_trace+0x1d/0xb0 [91655.404954] __kmalloc+0x203/0x250 [91655.405638] virtqueue_add+0x493/0xc70 [91655.406390] ? finish_wait+0x80/0x80 [91655.407097] virtqueue_add_sgs+0x80/0xa0 [91655.407852] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.408818] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.409752] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.410934] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.411969] scsi_queue_rq+0x512/0xb10 [91655.412727] blk_mq_dispatch_rq_list+0x154/0x880 [91655.413023] ll_ost_io00_057: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91655.413874] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91655.417168] __blk_mq_sched_dispatch_requests+0x110/0x140 [91655.418171] blk_mq_sched_dispatch_requests+0x30/0x60 [91655.419129] __blk_mq_run_hw_queue+0x2b/0x70 [91655.419945] __blk_mq_delay_run_hw_queue+0x141/0x160 [91655.420874] blk_mq_sched_insert_requests+0x6c/0xf0 [91655.421795] blk_mq_flush_plug_list+0x135/0x220 [91655.422648] blk_flush_plug_list+0xd7/0x100 [91655.423460] blk_finish_plug+0x25/0x36 [91655.424212] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.425284] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.426267] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.427546] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.429021] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.430117] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.431012] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.431831] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.432802] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.433738] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.434676] ? flush_work+0x42/0x1d0 [91655.435389] ? internal_add_timer+0x42/0x70 [91655.436205] ? _cond_resched+0x15/0x30 [91655.436954] ? mutex_lock+0xe/0x30 [91655.437637] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.438636] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.439754] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.440868] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.441777] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.442763] kthread+0x134/0x150 [91655.443474] ? set_kthread_struct+0x50/0x50 [91655.444555] ret_from_fork+0x1f/0x40 [91655.445406] warn_alloc_show_mem: 9 callbacks suppressed [91655.445409] Mem-Info: [91655.445407] CPU: 3 PID: 527434 Comm: ll_ost_io00_057 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.446503] active_anon:27195 inactive_anon:347773 isolated_anon:0 active_file:31245192 inactive_file:3199450 isolated_file:20 unevictable:0 dirty:4829 writeback:668 slab_reclaimable:773435 slab_unreclaimable:581092 mapped:23584 shmem:163 pagetables:5501 bounce:0 free:1842573 free_pcp:439 free_cma:0 [91655.447275] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.449643] Node 0 active_anon:108780kB inactive_anon:1391092kB active_file:124988996kB inactive_file:12797800kB unevictable:0kB isolated(anon):0kB isolated(file):80kB mapped:94336kB dirty:19316kB writeback:2672kB shmem:652kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35168kB pagetables:22004kB all_unreclaimable? no [91655.457744] Call Trace: [91655.459312] Node 0 [91655.464340] dump_stack+0x41/0x60 [91655.464941] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91655.465489] warn_alloc.cold.127+0x7b/0x108 [91655.466280] lowmem_reserve[]: [91655.469873] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.470741] 0 [91655.471395] __alloc_pages_nodemask+0x2e2/0x330 [91655.472264] 913 [91655.472741] kmalloc_order+0x28/0x90 [91655.473635] 150007 [91655.474149] kmalloc_order_trace+0x1d/0xb0 [91655.474897] 150007 [91655.475435] __kmalloc+0x203/0x250 [91655.476256] 150007 [91655.476805] virtqueue_add+0x493/0xc70 [91655.478061] ? finish_wait+0x80/0x80 [91655.478846] Node 0 [91655.479305] virtqueue_add_sgs+0x80/0xa0 [91655.480056] DMA32 free:596576kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:136kB inactive_file:4kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:20kB local_pcp:0kB free_cma:0kB [91655.480604] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.481398] lowmem_reserve[]: [91655.485218] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.486243] 0 [91655.486927] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.487837] 0 [91655.488305] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.489691] 149093 [91655.490161] scsi_queue_rq+0x512/0xb10 [91655.491696] 149093 [91655.492257] __blk_mq_try_issue_directly+0x163/0x200 [91655.493076] 149093 [91655.493654] blk_mq_request_issue_directly+0x4e/0xb0 [91655.495218] blk_mq_try_issue_list_directly+0x62/0x100 [91655.496195] Node 0 [91655.496689] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.497674] Normal free:6744148kB min:49268kB low:201940kB high:354612kB active_anon:108400kB inactive_anon:1386732kB active_file:124998056kB inactive_file:12797796kB unevictable:0kB writepending:21968kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1732kB local_pcp:152kB free_cma:0kB [91655.498231] blk_mq_flush_plug_list+0x135/0x220 [91655.499190] lowmem_reserve[]: [91655.503499] blk_flush_plug_list+0xd7/0x100 [91655.504410] 0 [91655.505091] blk_finish_plug+0x25/0x36 [91655.505932] 0 [91655.506422] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.507198] 0 [91655.507879] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.508922] 0 [91655.509444] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.510436] 0 [91655.510934] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.512549] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.513629] Node 0 [91655.514085] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.515057] DMA: [91655.515589] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.516505] 0*4kB [91655.517009] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.517804] 0*8kB [91655.518332] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.519188] 0*16kB [91655.519721] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.520553] 0*32kB [91655.521091] ? flush_work+0x42/0x1d0 [91655.521987] 0*64kB [91655.522527] ? internal_add_timer+0x42/0x70 [91655.523269] 0*128kB [91655.523816] ? _cond_resched+0x15/0x30 [91655.524665] 0*256kB [91655.525215] ? mutex_lock+0xe/0x30 [91655.525983] 0*512kB [91655.526526] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.527183] 1*1024kB [91655.527750] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.528655] (U) [91655.529227] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.530238] 1*2048kB [91655.530750] ? finish_wait+0x80/0x80 [91655.531801] (M) [91655.532351] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.533048] 2*4096kB [91655.533540] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.534535] (M) [91655.535100] kthread+0x134/0x150 [91655.536169] = 11264kB [91655.536827] ? set_kthread_struct+0x50/0x50 [91655.537522] Node 0 [91655.538293] ret_from_fork+0x1f/0x40 [91655.539120] DMA32: [91655.539773] CPU: 19 PID: 527692 Comm: ll_ost_io04_055 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.540528] 170*4kB [91655.541267] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.543327] (UM) [91655.544156] Call Trace: [91655.545937] 129*8kB [91655.546642] dump_stack+0x41/0x60 [91655.547240] (UMH) [91655.547808] warn_alloc.cold.127+0x7b/0x108 [91655.548503] 199*16kB [91655.549030] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.549867] (UMEH) [91655.550442] ? __alloc_pages_nodemask+0x166/0x330 [91655.551313] 125*32kB [91655.551858] __alloc_pages_nodemask+0x2e2/0x330 [91655.552755] (UMEH) [91655.553311] kmalloc_order+0x28/0x90 [91655.554173] 166*64kB [91655.554706] kmalloc_order_trace+0x1d/0xb0 [91655.555440] (UMEH) [91655.556201] __kmalloc+0x203/0x250 [91655.557272] 157*128kB [91655.557879] virtqueue_add+0x493/0xc70 [91655.558761] (UMEH) [91655.559564] ? finish_wait+0x80/0x80 [91655.560574] 134*256kB [91655.561315] virtqueue_add_sgs+0x80/0xa0 [91655.562183] (UMEH) [91655.562783] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.563552] 113*512kB [91655.564083] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.565081] (UM) [91655.565653] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.566508] 88*1024kB [91655.567021] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.567968] (UME) [91655.568549] scsi_queue_rq+0x512/0xb10 [91655.569582] 53*2048kB [91655.570119] __blk_mq_try_issue_directly+0x163/0x200 [91655.570886] (UM) [91655.571471] blk_mq_request_issue_directly+0x4e/0xb0 [91655.572408] 65*4096kB [91655.572923] blk_mq_try_issue_list_directly+0x62/0x100 [91655.573832] (UM) [91655.574392] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.575512] = 596672kB [91655.576147] blk_mq_flush_plug_list+0x135/0x220 [91655.577376] Node 0 [91655.578173] blk_flush_plug_list+0xd7/0x100 [91655.579401] Normal: [91655.580154] blk_finish_plug+0x25/0x36 [91655.581286] 543087*4kB [91655.582056] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.582859] (UME) [91655.583569] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.584618] 509184*8kB [91655.585149] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.586217] (UME) [91655.586821] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.587968] 28301*16kB [91655.588495] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.589583] (UMEH) [91655.590175] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.591128] 4*32kB [91655.591665] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.592548] (UH) [91655.593076] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.593852] 1*64kB [91655.594364] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.595212] (H) [91655.595755] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.596557] 0*128kB [91655.597055] ? flush_work+0x42/0x1d0 [91655.597935] 0*256kB [91655.598475] ? internal_add_timer+0x42/0x70 [91655.599205] 0*512kB [91655.599764] ? _cond_resched+0x15/0x30 [91655.600565] 0*1024kB [91655.601105] ? mutex_lock+0xe/0x30 [91655.601865] 0*2048kB [91655.602408] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.603103] 0*4096kB [91655.603668] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.604565] = 6698828kB [91655.605130] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.606199] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [91655.606827] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.607835] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [91655.609317] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.610133] 34467449 total pagecache pages [91655.611553] kthread+0x134/0x150 [91655.612488] 2131 pages in swap cache [91655.613299] ? set_kthread_struct+0x50/0x50 [91655.613987] Swap cache stats: add 1308918, delete 1306804, find 55539/66791 [91655.614859] ret_from_fork+0x1f/0x40 [91655.615911] Free swap = 6293976kB [91655.617646] CPU: 1 PID: 527271 Comm: ll_ost_io00_033 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.618736] Total swap = 11075580kB [91655.619598] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.622704] 39321462 pages RAM [91655.623601] Call Trace: [91655.625849] 0 pages HighMem/MovableOnly [91655.626855] dump_stack+0x41/0x60 [91655.627810] 905848 pages reserved [91655.628779] warn_alloc.cold.127+0x7b/0x108 [91655.629835] 0 pages hwpoisoned [91655.630683] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.633786] ? __alloc_pages_nodemask+0x166/0x330 [91655.634850] __alloc_pages_nodemask+0x2e2/0x330 [91655.635864] kmalloc_order+0x28/0x90 [91655.636680] kmalloc_order_trace+0x1d/0xb0 [91655.637598] __kmalloc+0x203/0x250 [91655.638386] virtqueue_add+0x493/0xc70 [91655.639277] ? finish_wait+0x80/0x80 [91655.640127] virtqueue_add_sgs+0x80/0xa0 [91655.641036] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.642169] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.643127] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.644204] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.645425] scsi_queue_rq+0x512/0xb10 [91655.646291] __blk_mq_try_issue_directly+0x163/0x200 [91655.647345] blk_mq_request_issue_directly+0x4e/0xb0 [91655.648412] blk_mq_try_issue_list_directly+0x62/0x100 [91655.649499] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.650546] blk_mq_flush_plug_list+0x135/0x220 [91655.651537] blk_flush_plug_list+0xd7/0x100 [91655.652466] blk_finish_plug+0x25/0x36 [91655.653312] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.654548] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.655675] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.657011] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.658300] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.659416] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.660456] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.661401] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.662534] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.663626] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.664767] ? flush_work+0x42/0x1d0 [91655.665600] ? internal_add_timer+0x42/0x70 [91655.666510] ? _cond_resched+0x15/0x30 [91655.667372] ? mutex_lock+0xe/0x30 [91655.668156] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.669312] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.670575] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.671858] ? finish_wait+0x80/0x80 [91655.672673] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.673735] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.674869] kthread+0x134/0x150 [91655.675625] ? set_kthread_struct+0x50/0x50 [91655.676520] ret_from_fork+0x1f/0x40 [91655.677481] ll_ost_io01_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91655.680238] CPU: 5 PID: 527029 Comm: ll_ost_io01_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.683090] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.684768] Call Trace: [91655.685491] dump_stack+0x41/0x60 [91655.686265] warn_alloc.cold.127+0x7b/0x108 [91655.686306] ll_ost_io04_048: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [91655.687067] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.687070] ,mems_allowed=0 [91655.687073] ? __alloc_pages_nodemask+0x166/0x330 [91655.690513] __alloc_pages_nodemask+0x2e2/0x330 [91655.693755] kmalloc_order+0x28/0x90 [91655.694474] kmalloc_order_trace+0x1d/0xb0 [91655.695251] __kmalloc+0x203/0x250 [91655.695649] ll_ost_io04_075: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91655.695939] virtqueue_add+0x493/0xc70 [91655.698855] virtqueue_add_sgs+0x80/0xa0 [91655.699642] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.700934] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.702080] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.703355] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.704780] scsi_queue_rq+0x512/0xb10 [91655.705806] __blk_mq_try_issue_directly+0x163/0x200 [91655.706769] ll_ost_io02_075: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91655.706835] ll_ost_io02_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91655.706852] ll_ost_io02_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91655.707041] blk_mq_request_issue_directly+0x4e/0xb0 [91655.709064] ,cpuset= [91655.711920] blk_mq_try_issue_list_directly+0x62/0x100 [91655.714280] / [91655.715312] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.715987] ,mems_allowed=0 [91655.717004] blk_mq_flush_plug_list+0x135/0x220 [91655.718563] blk_flush_plug_list+0xd7/0x100 [91655.721644] blk_finish_plug+0x25/0x36 [91655.722504] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.723719] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.724722] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.725855] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.726951] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.727924] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.728832] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.729643] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.730586] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.731504] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.732429] ? flush_work+0x42/0x1d0 [91655.733135] ? internal_add_timer+0x42/0x70 [91655.733953] ? _cond_resched+0x15/0x30 [91655.734698] ? mutex_lock+0xe/0x30 [91655.735381] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.736371] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.737449] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.738580] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.739505] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.740500] kthread+0x134/0x150 [91655.741147] ? set_kthread_struct+0x50/0x50 [91655.741927] ret_from_fork+0x1f/0x40 [91655.742609] CPU: 9 PID: 527866 Comm: ll_ost_io02_075 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.746244] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.748021] Call Trace: [91655.748677] dump_stack+0x41/0x60 [91655.749476] warn_alloc.cold.127+0x7b/0x108 [91655.750407] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.751389] __alloc_pages_nodemask+0x2e2/0x330 [91655.752356] kmalloc_order+0x28/0x90 [91655.753173] kmalloc_order_trace+0x1d/0xb0 [91655.754067] __kmalloc+0x203/0x250 [91655.754874] virtqueue_add+0x493/0xc70 [91655.755731] ? finish_wait+0x80/0x80 [91655.756549] virtqueue_add_sgs+0x80/0xa0 [91655.757434] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.758554] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.759492] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.760537] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.761706] scsi_queue_rq+0x512/0xb10 [91655.762555] __blk_mq_try_issue_directly+0x163/0x200 [91655.763603] blk_mq_request_issue_directly+0x4e/0xb0 [91655.764628] blk_mq_try_issue_list_directly+0x62/0x100 [91655.766178] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.767209] blk_mq_flush_plug_list+0x135/0x220 [91655.768180] blk_flush_plug_list+0xd7/0x100 [91655.769078] blk_finish_plug+0x25/0x36 [91655.769908] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.771135] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.772266] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.773569] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.774835] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.775933] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.776952] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.777869] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.778950] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.780006] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.781085] ? flush_work+0x42/0x1d0 [91655.781884] ? internal_add_timer+0x42/0x70 [91655.782771] ? _cond_resched+0x15/0x30 [91655.783605] ? mutex_lock+0xe/0x30 [91655.784392] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.785571] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.786849] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.788119] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.789164] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.790332] kthread+0x134/0x150 [91655.791088] ? set_kthread_struct+0x50/0x50 [91655.791985] ret_from_fork+0x1f/0x40 [91655.792786] CPU: 8 PID: 527337 Comm: ll_ost_io02_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.795936] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.798077] Call Trace: [91655.798706] dump_stack+0x41/0x60 [91655.799419] warn_alloc.cold.127+0x7b/0x108 [91655.800378] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.801362] __alloc_pages_nodemask+0x2e2/0x330 [91655.802305] kmalloc_order+0x28/0x90 [91655.803170] kmalloc_order_trace+0x1d/0xb0 [91655.804153] __kmalloc+0x203/0x250 [91655.804969] virtqueue_add+0x493/0xc70 [91655.805926] ? finish_wait+0x80/0x80 [91655.806705] virtqueue_add_sgs+0x80/0xa0 [91655.807432] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.808374] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.809176] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.810070] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.811065] scsi_queue_rq+0x512/0xb10 [91655.811786] __blk_mq_try_issue_directly+0x163/0x200 [91655.812664] blk_mq_request_issue_directly+0x4e/0xb0 [91655.813537] blk_mq_try_issue_list_directly+0x62/0x100 [91655.814454] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.815320] blk_mq_flush_plug_list+0x135/0x220 [91655.816142] blk_flush_plug_list+0xd7/0x100 [91655.816910] blk_finish_plug+0x25/0x36 [91655.817611] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.818638] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.819578] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.820726] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.821812] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.822764] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.823628] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.824416] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.825335] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.826220] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.827178] ? flush_work+0x42/0x1d0 [91655.827881] ? internal_add_timer+0x42/0x70 [91655.828787] ? _cond_resched+0x15/0x30 [91655.829847] ? mutex_lock+0xe/0x30 [91655.830550] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.831533] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.832607] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.833682] ? lprocfs_counter_add+0x86/0x140 [obdclass] [91655.834700] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.835604] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.836571] kthread+0x134/0x150 [91655.837211] ? set_kthread_struct+0x50/0x50 [91655.837982] ret_from_fork+0x1f/0x40 [91655.838663] CPU: 16 PID: 528063 Comm: ll_ost_io04_075 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.840994] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.843097] Call Trace: [91655.843899] dump_stack+0x41/0x60 [91655.844854] warn_alloc.cold.127+0x7b/0x108 [91655.845941] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.847039] __alloc_pages_nodemask+0x2e2/0x330 [91655.847876] kmalloc_order+0x28/0x90 [91655.848576] kmalloc_order_trace+0x1d/0xb0 [91655.849330] __kmalloc+0x203/0x250 [91655.850003] virtqueue_add+0x493/0xc70 [91655.850755] ? finish_wait+0x80/0x80 [91655.851524] virtqueue_add_sgs+0x80/0xa0 [91655.852299] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.853263] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.854072] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.854974] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.856012] scsi_queue_rq+0x512/0xb10 [91655.857047] __blk_mq_try_issue_directly+0x163/0x200 [91655.857992] blk_mq_request_issue_directly+0x4e/0xb0 [91655.858886] blk_mq_try_issue_list_directly+0x62/0x100 [91655.859851] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.860762] blk_mq_flush_plug_list+0x135/0x220 [91655.861611] blk_flush_plug_list+0xd7/0x100 [91655.862396] blk_finish_plug+0x25/0x36 [91655.863116] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.864181] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.865181] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.866298] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.867388] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.868340] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.869257] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.870082] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.871044] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.871958] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.872903] ? flush_work+0x42/0x1d0 [91655.873609] ? internal_add_timer+0x42/0x70 [91655.874390] ? _cond_resched+0x15/0x30 [91655.875118] ? mutex_lock+0xe/0x30 [91655.875809] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.876815] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.877934] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.879038] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.879960] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.880948] kthread+0x134/0x150 [91655.881591] ? set_kthread_struct+0x50/0x50 [91655.882370] ret_from_fork+0x1f/0x40 [91655.883062] CPU: 19 PID: 527685 Comm: ll_ost_io04_048 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.885822] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.887910] Call Trace: [91655.888672] dump_stack+0x41/0x60 [91655.889600] warn_alloc.cold.127+0x7b/0x108 [91655.890691] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.891862] ? __alloc_pages_nodemask+0x166/0x330 [91655.893063] __alloc_pages_nodemask+0x2e2/0x330 [91655.894229] kmalloc_order+0x28/0x90 [91655.895225] kmalloc_order_trace+0x1d/0xb0 [91655.896311] __kmalloc+0x203/0x250 [91655.897263] virtqueue_add+0x493/0xc70 [91655.898277] virtqueue_add_sgs+0x80/0xa0 [91655.899334] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.900673] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.901815] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.903074] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.904433] scsi_queue_rq+0x512/0xb10 [91655.905417] __blk_mq_try_issue_directly+0x163/0x200 [91655.906637] blk_mq_request_issue_directly+0x4e/0xb0 [91655.907841] blk_mq_try_issue_list_directly+0x62/0x100 [91655.909075] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.910238] blk_mq_flush_plug_list+0x135/0x220 [91655.911327] blk_flush_plug_list+0xd7/0x100 [91655.912388] blk_finish_plug+0x25/0x36 [91655.913363] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.914797] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.916140] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.917690] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.919190] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.920507] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.921716] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.922831] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.924139] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.925424] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.926721] ? flush_work+0x42/0x1d0 [91655.927671] ? internal_add_timer+0x42/0x70 [91655.928732] ? _cond_resched+0x15/0x30 [91655.929708] ? mutex_lock+0xe/0x30 [91655.930676] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91655.932064] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91655.933577] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91655.934874] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91655.936019] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91655.937364] kthread+0x134/0x150 [91655.938241] ? set_kthread_struct+0x50/0x50 [91655.939283] ret_from_fork+0x1f/0x40 [91655.940226] CPU: 10 PID: 527071 Comm: ll_ost_io02_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91655.943471] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91655.946046] Call Trace: [91655.946994] dump_stack+0x41/0x60 [91655.948108] warn_alloc.cold.127+0x7b/0x108 [91655.949962] __alloc_pages_slowpath+0xcb2/0xcd0 [91655.951446] ? __alloc_pages_nodemask+0x166/0x330 [91655.953058] __alloc_pages_nodemask+0x2e2/0x330 [91655.954581] kmalloc_order+0x28/0x90 [91655.955808] kmalloc_order_trace+0x1d/0xb0 [91655.957063] __kmalloc+0x203/0x250 [91655.958238] virtqueue_add+0x493/0xc70 [91655.959617] ? finish_wait+0x80/0x80 [91655.960843] virtqueue_add_sgs+0x80/0xa0 [91655.962095] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91655.963683] ? scsi_alloc_sgtables+0x84/0x1c0 [91655.965121] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91655.966689] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91655.968460] scsi_queue_rq+0x512/0xb10 [91655.969731] __blk_mq_try_issue_directly+0x163/0x200 [91655.971346] blk_mq_request_issue_directly+0x4e/0xb0 [91655.973041] blk_mq_try_issue_list_directly+0x62/0x100 [91655.975302] blk_mq_sched_insert_requests+0xa4/0xf0 [91655.976942] blk_mq_flush_plug_list+0x135/0x220 [91655.979172] blk_flush_plug_list+0xd7/0x100 [91655.980955] blk_finish_plug+0x25/0x36 [91655.982576] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91655.984859] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91655.986336] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.987894] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91655.989432] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91655.990781] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91655.991997] ofd_commitrw+0x5b4/0xd20 [ofd] [91655.993109] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.994431] obd_commitrw+0x1b6/0x370 [ptlrpc] [91655.995698] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91655.997052] ? flush_work+0x42/0x1d0 [91655.998021] ? internal_add_timer+0x42/0x70 [91655.999081] ? _cond_resched+0x15/0x30 [91656.000096] ? mutex_lock+0xe/0x30 [91656.001011] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91656.002381] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91656.003845] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91656.005309] ? lu_context_exit+0x3b/0x70 [obdclass] [91656.006548] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91656.007763] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91656.009120] kthread+0x134/0x150 [91656.009978] ? set_kthread_struct+0x50/0x50 [91656.011060] ret_from_fork+0x1f/0x40 [91661.259603] warn_alloc: 2497 callbacks suppressed [91661.259607] ll_ost_io03_028: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91661.260107] ll_ost_io04_006: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91661.260660] ll_ost_io03_049: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91661.260672] CPU: 14 PID: 527326 Comm: ll_ost_io03_049 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91661.260676] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91661.260677] Call Trace: [91661.260682] dump_stack+0x41/0x60 [91661.260690] warn_alloc.cold.127+0x7b/0x108 [91661.260697] __alloc_pages_slowpath+0xcb2/0xcd0 [91661.260700] ? __alloc_pages_nodemask+0x166/0x330 [91661.260704] __alloc_pages_nodemask+0x2e2/0x330 [91661.260707] kmalloc_order+0x28/0x90 [91661.260713] kmalloc_order_trace+0x1d/0xb0 [91661.260716] __kmalloc+0x203/0x250 [91661.260722] virtqueue_add+0x493/0xc70 [91661.260729] ? finish_wait+0x80/0x80 [91661.260733] virtqueue_add_sgs+0x80/0xa0 [91661.260738] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91661.260744] ? scsi_alloc_sgtables+0x84/0x1c0 [91661.260750] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91661.260754] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91661.260758] scsi_queue_rq+0x512/0xb10 [91661.260761] blk_mq_dispatch_rq_list+0x154/0x880 [91661.260768] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91661.260772] __blk_mq_sched_dispatch_requests+0x110/0x140 [91661.260775] blk_mq_sched_dispatch_requests+0x30/0x60 [91661.260778] __blk_mq_run_hw_queue+0x2b/0x70 [91661.260781] __blk_mq_delay_run_hw_queue+0x141/0x160 [91661.260785] blk_mq_sched_insert_requests+0x6c/0xf0 [91661.260788] blk_mq_flush_plug_list+0x135/0x220 [91661.260796] blk_flush_plug_list+0xd7/0x100 [91661.260801] blk_finish_plug+0x25/0x36 [91661.260806] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91661.260843] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91661.260874] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.260901] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.260928] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91661.260956] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91661.260976] ofd_commitrw+0x5b4/0xd20 [ofd] [91661.260989] ,cpuset= [91661.260995] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.261131] obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.261253] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91661.261374] ? flush_work+0x42/0x1d0 [91661.261380] ? internal_add_timer+0x42/0x70 [91661.261385] ? _cond_resched+0x15/0x30 [91661.261388] ? mutex_lock+0xe/0x30 [91661.261392] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91661.261502] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91661.261624] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91661.261741] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91661.261860] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91661.261964] kthread+0x134/0x150 [91661.261969] ? set_kthread_struct+0x50/0x50 [91661.261972] ret_from_fork+0x1f/0x40 [91661.261978] warn_alloc_show_mem: 9 callbacks suppressed [91661.261980] Mem-Info: [91661.261984] active_anon:27195 inactive_anon:348176 isolated_anon:0 active_file:31214317 inactive_file:2693508 isolated_file:0 unevictable:0 dirty:3347 writeback:341 slab_reclaimable:773602 slab_unreclaimable:594582 mapped:23911 shmem:163 pagetables:5503 bounce:0 free:2363824 free_pcp:463 free_cma:0 [91661.261990] Node 0 active_anon:108780kB inactive_anon:1392704kB active_file:124857268kB inactive_file:10774032kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:95644kB dirty:13388kB writeback:1364kB shmem:652kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35168kB pagetables:22012kB all_unreclaimable? no [91661.261998] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91661.262004] lowmem_reserve[]: 0 913 150007 150007 150007 [91661.262010] Node 0 DMA32 free:596516kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:136kB inactive_file:4kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91661.262016] lowmem_reserve[]: 0 0 149093 149093 149093 [91661.262020] Node 0 Normal free:8847516kB min:49268kB low:201940kB high:354612kB active_anon:108400kB inactive_anon:1388344kB active_file:124856948kB inactive_file:10774028kB unevictable:0kB writepending:14752kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1852kB local_pcp:160kB free_cma:0kB [91661.262027] lowmem_reserve[]: 0 0 0 0 0 [91661.262030] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [91661.262043] Node 0 DMA32: 169*4kB (UMH) 128*8kB (UMH) 198*16kB (UMEH) 118*32kB (UME) 164*64kB (UMEH) 156*128kB (UMEH) 133*256kB (UME) 114*512kB (UMH) 88*1024kB (UMEH) 53*2048kB (UM) 65*4096kB (UM) = 596420kB [91661.262064] Node 0 Normal: 876398*4kB (UME) 592473*8kB (UME) 37693*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 8848464kB [91661.262080] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [91661.262083] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [91661.262085] 33907966 total pagecache pages [91661.262086] 2156 pages in swap cache [91661.262087] Swap cache stats: add 1309008, delete 1306869, find 55570/66882 [91661.262088] Free swap = 6294232kB [91661.262089] Total swap = 11075580kB [91661.262091] 39321462 pages RAM [91661.262091] 0 pages HighMem/MovableOnly [91661.262092] 905848 pages reserved [91661.262093] 0 pages hwpoisoned [91661.263916] ,cpuset= [91661.265678] / [91661.274502] ll_ost_io01_021: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91661.275050] / [91661.275050] ,mems_allowed=0 [91661.275054] CPU: 12 PID: 527228 Comm: ll_ost_io03_028 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91661.275057] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91661.275058] Call Trace: [91661.275062] dump_stack+0x41/0x60 [91661.275068] warn_alloc.cold.127+0x7b/0x108 [91661.275074] __alloc_pages_slowpath+0xcb2/0xcd0 [91661.275076] ? __alloc_pages_nodemask+0x166/0x330 [91661.275079] __alloc_pages_nodemask+0x2e2/0x330 [91661.275081] kmalloc_order+0x28/0x90 [91661.275086] kmalloc_order_trace+0x1d/0xb0 [91661.275089] __kmalloc+0x203/0x250 [91661.275093] ? __find_get_block+0x23a/0x2b0 [91661.275099] virtqueue_add+0x493/0xc70 [91661.275104] ? finish_wait+0x80/0x80 [91661.275107] virtqueue_add_sgs+0x80/0xa0 [91661.275111] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91661.275116] ? scsi_alloc_sgtables+0x84/0x1c0 [91661.275120] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91661.275123] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91661.275125] scsi_queue_rq+0x512/0xb10 [91661.275127] blk_mq_dispatch_rq_list+0x154/0x880 [91661.275133] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91661.275136] __blk_mq_sched_dispatch_requests+0x110/0x140 [91661.275139] blk_mq_sched_dispatch_requests+0x30/0x60 [91661.275140] __blk_mq_run_hw_queue+0x2b/0x70 [91661.275143] __blk_mq_delay_run_hw_queue+0x141/0x160 [91661.275146] blk_mq_sched_insert_requests+0x6c/0xf0 [91661.275148] blk_mq_flush_plug_list+0x135/0x220 [91661.275152] blk_flush_plug_list+0xd7/0x100 [91661.275156] blk_finish_plug+0x25/0x36 [91661.275160] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91661.275190] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91661.275213] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.275232] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.275251] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91661.275270] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91661.275285] ofd_commitrw+0x5b4/0xd20 [ofd] [91661.275296] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.275388] obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.275457] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91661.275526] ? flush_work+0x42/0x1d0 [91661.275530] ? internal_add_timer+0x42/0x70 [91661.275533] ? _cond_resched+0x15/0x30 [91661.275535] ? mutex_lock+0xe/0x30 [91661.275538] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91661.275606] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91661.275678] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91661.275747] ? finish_wait+0x80/0x80 [91661.275749] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91661.275835] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91661.275896] kthread+0x134/0x150 [91661.275899] ,cpuset=/ [91661.276693] ,mems_allowed=0 [91661.277528] ? set_kthread_struct+0x50/0x50 [91661.278508] ,mems_allowed=0 [91661.279287] ret_from_fork+0x1f/0x40 [91661.281158] CPU: 19 PID: 527167 Comm: ll_ost_io04_006 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91661.319859] ll_ost_io04_000: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91661.320517] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91661.321618] ,cpuset= [91661.322445] Call Trace: [91661.323602] / [91661.324290] dump_stack+0x41/0x60 [91661.328576] ll_ost_io04_010: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [91661.330921] ,mems_allowed=0 [91661.336570] warn_alloc.cold.127+0x7b/0x108 [91661.340656] ,mems_allowed=0 [91661.341783] __alloc_pages_slowpath+0xcb2/0xcd0 [91661.347363] ? __alloc_pages_nodemask+0x166/0x330 [91661.452076] __alloc_pages_nodemask+0x2e2/0x330 [91661.453044] kmalloc_order+0x28/0x90 [91661.453857] kmalloc_order_trace+0x1d/0xb0 [91661.454763] __kmalloc+0x203/0x250 [91661.455544] virtqueue_add+0x493/0xc70 [91661.456388] ? finish_wait+0x80/0x80 [91661.457187] virtqueue_add_sgs+0x80/0xa0 [91661.458067] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91661.459171] ? scsi_alloc_sgtables+0x84/0x1c0 [91661.460106] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91661.461157] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91661.462299] scsi_queue_rq+0x512/0xb10 [91661.463129] blk_mq_dispatch_rq_list+0x154/0x880 [91661.464097] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91661.465054] __blk_mq_sched_dispatch_requests+0x110/0x140 [91661.466164] blk_mq_sched_dispatch_requests+0x30/0x60 [91661.467215] __blk_mq_run_hw_queue+0x2b/0x70 [91661.468138] __blk_mq_delay_run_hw_queue+0x141/0x160 [91661.469169] blk_mq_sched_insert_requests+0x6c/0xf0 [91661.470193] blk_mq_flush_plug_list+0x135/0x220 [91661.471164] blk_flush_plug_list+0xd7/0x100 [91661.472071] blk_finish_plug+0x25/0x36 [91661.472916] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91661.474120] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91661.475229] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.476528] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.477788] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91661.478896] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91661.479930] ofd_commitrw+0x5b4/0xd20 [ofd] [91661.480854] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.481964] obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.483039] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91661.484144] ? flush_work+0x42/0x1d0 [91661.484964] ? internal_add_timer+0x42/0x70 [91661.485872] ? _cond_resched+0x15/0x30 [91661.486696] ? mutex_lock+0xe/0x30 [91661.487466] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91661.488622] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91661.489858] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91661.491107] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91661.492141] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91661.493281] kthread+0x134/0x150 [91661.494032] ? set_kthread_struct+0x50/0x50 [91661.494922] ret_from_fork+0x1f/0x40 [91661.495710] CPU: 18 PID: 527191 Comm: ll_ost_io04_010 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91661.499944] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91661.501635] Call Trace: [91661.502173] dump_stack+0x41/0x60 [91661.502865] warn_alloc.cold.127+0x7b/0x108 [91661.503720] __alloc_pages_slowpath+0xcb2/0xcd0 [91661.504614] ? __alloc_pages_nodemask+0x166/0x330 [91661.505474] __alloc_pages_nodemask+0x2e2/0x330 [91661.506310] kmalloc_order+0x28/0x90 [91661.507010] kmalloc_order_trace+0x1d/0xb0 [91661.507791] __kmalloc+0x203/0x250 [91661.508482] virtqueue_add+0x493/0xc70 [91661.509355] virtqueue_add_sgs+0x80/0xa0 [91661.510404] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91661.511775] ? scsi_alloc_sgtables+0x84/0x1c0 [91661.512934] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91661.514212] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91661.515600] scsi_queue_rq+0x512/0xb10 [91661.516623] blk_mq_dispatch_rq_list+0x154/0x880 [91661.517697] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91661.518857] __blk_mq_sched_dispatch_requests+0x110/0x140 [91661.520171] blk_mq_sched_dispatch_requests+0x30/0x60 [91661.521187] __blk_mq_run_hw_queue+0x2b/0x70 [91661.521982] __blk_mq_delay_run_hw_queue+0x141/0x160 [91661.522876] blk_mq_sched_insert_requests+0x6c/0xf0 [91661.523754] blk_mq_flush_plug_list+0x135/0x220 [91661.524580] blk_flush_plug_list+0xd7/0x100 [91661.525346] blk_finish_plug+0x25/0x36 [91661.526056] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91661.527103] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91661.528066] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.529162] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.530236] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91661.531164] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91661.532038] ofd_commitrw+0x5b4/0xd20 [ofd] [91661.532822] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.533763] obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.534686] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91661.535653] ? flush_work+0x42/0x1d0 [91661.536364] ? internal_add_timer+0x42/0x70 [91661.537143] ? _cond_resched+0x15/0x30 [91661.537856] ? mutex_lock+0xe/0x30 [91661.538529] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91661.539496] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91661.540901] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91661.542014] ? finish_wait+0x80/0x80 [91661.542705] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91661.543580] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91661.544566] kthread+0x134/0x150 [91661.545251] ? set_kthread_struct+0x50/0x50 [91661.546059] ret_from_fork+0x1f/0x40 [91661.546899] CPU: 5 PID: 527078 Comm: ll_ost_io01_021 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91661.549914] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91661.552168] Call Trace: [91661.553016] dump_stack+0x41/0x60 [91661.554371] warn_alloc.cold.127+0x7b/0x108 [91661.555572] __alloc_pages_slowpath+0xcb2/0xcd0 [91661.556818] ? __alloc_pages_nodemask+0x166/0x330 [91661.558098] __alloc_pages_nodemask+0x2e2/0x330 [91661.559341] kmalloc_order+0x28/0x90 [91661.560366] kmalloc_order_trace+0x1d/0xb0 [91661.561511] __kmalloc+0x203/0x250 [91661.562815] virtqueue_add+0x493/0xc70 [91661.563805] ? finish_wait+0x80/0x80 [91661.564783] virtqueue_add_sgs+0x80/0xa0 [91661.565845] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91661.567051] ? scsi_alloc_sgtables+0x84/0x1c0 [91661.567859] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91661.568745] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91661.569728] scsi_queue_rq+0x512/0xb10 [91661.570486] blk_mq_dispatch_rq_list+0x154/0x880 [91661.571394] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91661.572553] __blk_mq_sched_dispatch_requests+0x110/0x140 [91661.573894] blk_mq_sched_dispatch_requests+0x30/0x60 [91661.575167] __blk_mq_run_hw_queue+0x2b/0x70 [91661.576274] __blk_mq_delay_run_hw_queue+0x141/0x160 [91661.577512] blk_mq_sched_insert_requests+0x6c/0xf0 [91661.578712] blk_mq_flush_plug_list+0x135/0x220 [91661.579869] blk_flush_plug_list+0xd7/0x100 [91661.580954] blk_finish_plug+0x25/0x36 [91661.581943] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91661.583352] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.584870] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.586374] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91661.587674] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91661.588912] ofd_commitrw+0x5b4/0xd20 [ofd] [91661.590060] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.591406] obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.592697] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91661.594045] ? flush_work+0x42/0x1d0 [91661.595056] ? internal_add_timer+0x42/0x70 [91661.596185] ? _cond_resched+0x15/0x30 [91661.597211] ? mutex_lock+0xe/0x30 [91661.598161] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91661.599528] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91661.600830] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91661.602116] ? finish_wait+0x80/0x80 [91661.602987] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91661.604116] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91661.605100] kthread+0x134/0x150 [91661.605749] ? set_kthread_struct+0x50/0x50 [91661.606523] ret_from_fork+0x1f/0x40 [91661.607417] CPU: 17 PID: 188109 Comm: ll_ost_io04_000 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91661.610387] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91661.612480] Call Trace: [91661.613282] dump_stack+0x41/0x60 [91661.614262] warn_alloc.cold.127+0x7b/0x108 [91661.615429] __alloc_pages_slowpath+0xcb2/0xcd0 [91661.616645] ? __alloc_pages_nodemask+0x166/0x330 [91661.617881] __alloc_pages_nodemask+0x2e2/0x330 [91661.619080] kmalloc_order+0x28/0x90 [91661.620099] kmalloc_order_trace+0x1d/0xb0 [91661.621247] __kmalloc+0x203/0x250 [91661.622199] virtqueue_add+0x493/0xc70 [91661.623206] ? finish_wait+0x80/0x80 [91661.624155] virtqueue_add_sgs+0x80/0xa0 [91661.625178] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91661.626486] ? scsi_alloc_sgtables+0x84/0x1c0 [91661.627608] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91661.628848] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91661.630218] scsi_queue_rq+0x512/0xb10 [91661.631213] __blk_mq_try_issue_directly+0x163/0x200 [91661.632442] blk_mq_request_issue_directly+0x4e/0xb0 [91661.633649] blk_mq_try_issue_list_directly+0x62/0x100 [91661.634691] blk_mq_sched_insert_requests+0xa4/0xf0 [91661.635915] blk_mq_flush_plug_list+0x135/0x220 [91661.637074] blk_flush_plug_list+0xd7/0x100 [91661.638183] blk_finish_plug+0x25/0x36 [91661.639181] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91661.640586] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91661.641935] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.643499] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91661.644974] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91661.646285] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91661.647501] ofd_commitrw+0x5b4/0xd20 [ofd] [91661.648621] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.649879] obd_commitrw+0x1b6/0x370 [ptlrpc] [91661.651131] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91661.652401] ? flush_work+0x42/0x1d0 [91661.653395] ? internal_add_timer+0x42/0x70 [91661.654516] ? _cond_resched+0x15/0x30 [91661.655533] ? mutex_lock+0xe/0x30 [91661.656460] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91661.657833] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91661.659325] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91661.660834] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91661.662103] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91661.663453] kthread+0x134/0x150 [91661.664196] ? set_kthread_struct+0x50/0x50 [91661.664991] ret_from_fork+0x1f/0x40 [91662.101636] ll_ost_io02_007: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91662.101703] ll_ost_io04_042: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91662.101708] ll_ost_io04_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91662.101718] CPU: 17 PID: 527995 Comm: ll_ost_io04_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91662.101722] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91662.101723] Call Trace: [91662.101728] dump_stack+0x41/0x60 [91662.101736] warn_alloc.cold.127+0x7b/0x108 [91662.101742] __alloc_pages_slowpath+0xcb2/0xcd0 [91662.101746] ? __alloc_pages_nodemask+0x166/0x330 [91662.101749] __alloc_pages_nodemask+0x2e2/0x330 [91662.101751] kmalloc_order+0x28/0x90 [91662.101757] kmalloc_order_trace+0x1d/0xb0 [91662.101760] __kmalloc+0x203/0x250 [91662.101765] virtqueue_add+0x493/0xc70 [91662.101772] ll_ost_io04_044: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91662.101771] ? finish_wait+0x80/0x80 [91662.101775] ,cpuset=/ [91662.101776] virtqueue_add_sgs+0x80/0xa0 [91662.101777] ,mems_allowed=0 [91662.101782] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91662.101788] ? scsi_alloc_sgtables+0x84/0x1c0 [91662.101794] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91662.101798] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91662.101802] scsi_queue_rq+0x512/0xb10 [91662.101810] __blk_mq_try_issue_directly+0x163/0x200 [91662.101817] blk_mq_request_issue_directly+0x4e/0xb0 [91662.101821] blk_mq_try_issue_list_directly+0x62/0x100 [91662.101825] blk_mq_sched_insert_requests+0xa4/0xf0 [91662.101829] blk_mq_flush_plug_list+0x135/0x220 [91662.101834] blk_flush_plug_list+0xd7/0x100 [91662.101840] blk_finish_plug+0x25/0x36 [91662.101846] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91662.101885] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91662.101915] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.101942] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.101971] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91662.101997] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91662.102019] ofd_commitrw+0x5b4/0xd20 [ofd] [91662.102037] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.102166] obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.102272] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91662.102383] ? flush_work+0x42/0x1d0 [91662.102389] ? internal_add_timer+0x42/0x70 [91662.102393] ? _cond_resched+0x15/0x30 [91662.102396] ? mutex_lock+0xe/0x30 [91662.102401] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91662.102517] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91662.102635] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91662.102728] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91662.102790] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91662.102887] kthread+0x134/0x150 [91662.102892] ? set_kthread_struct+0x50/0x50 [91662.102895] ret_from_fork+0x1f/0x40 [91662.102902] CPU: 19 PID: 527681 Comm: ll_ost_io04_044 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91662.102906] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91662.102908] Call Trace: [91662.102911] dump_stack+0x41/0x60 [91662.102917] warn_alloc.cold.127+0x7b/0x108 [91662.102923] __alloc_pages_slowpath+0xcb2/0xcd0 [91662.102926] ? __alloc_pages_nodemask+0x166/0x330 [91662.102929] __alloc_pages_nodemask+0x2e2/0x330 [91662.102932] kmalloc_order+0x28/0x90 [91662.102936] kmalloc_order_trace+0x1d/0xb0 [91662.102940] __kmalloc+0x203/0x250 [91662.102944] virtqueue_add+0x493/0xc70 [91662.102950] ? finish_wait+0x80/0x80 [91662.102953] virtqueue_add_sgs+0x80/0xa0 [91662.102957] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91662.102961] ? scsi_alloc_sgtables+0x84/0x1c0 [91662.102966] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91662.102969] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91662.102972] scsi_queue_rq+0x512/0xb10 [91662.102975] blk_mq_dispatch_rq_list+0x154/0x880 [91662.102981] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91662.102984] __blk_mq_sched_dispatch_requests+0x110/0x140 [91662.102987] blk_mq_sched_dispatch_requests+0x30/0x60 [91662.102989] __blk_mq_run_hw_queue+0x2b/0x70 [91662.102992] __blk_mq_delay_run_hw_queue+0x141/0x160 [91662.102995] blk_mq_sched_insert_requests+0x6c/0xf0 [91662.102997] blk_mq_flush_plug_list+0x135/0x220 [91662.103001] blk_flush_plug_list+0xd7/0x100 [91662.103005] blk_finish_plug+0x25/0x36 [91662.103009] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91662.103043] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91662.103073] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.103098] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.103123] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91662.103148] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91662.103166] ofd_commitrw+0x5b4/0xd20 [ofd] [91662.103182] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.103292] obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.103390] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91662.103494] ? flush_work+0x42/0x1d0 [91662.103499] ? internal_add_timer+0x42/0x70 [91662.103501] ? _cond_resched+0x15/0x30 [91662.103503] ? mutex_lock+0xe/0x30 [91662.103506] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91662.103607] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91662.103705] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91662.103804] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91662.103919] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91662.104026] kthread+0x134/0x150 [91662.104029] ? set_kthread_struct+0x50/0x50 [91662.104032] ret_from_fork+0x1f/0x40 [91662.104289] CPU: 10 PID: 527024 Comm: ll_ost_io02_007 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91662.106667] ,cpuset= [91662.109933] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91662.113071] / [91662.114728] Call Trace: [91662.115648] ,mems_allowed=0 [91662.116386] dump_stack+0x41/0x60 [91662.118150] warn_alloc.cold.127+0x7b/0x108 [91662.207312] __alloc_pages_slowpath+0xcb2/0xcd0 [91662.208120] ? __alloc_pages_nodemask+0x166/0x330 [91662.208939] __alloc_pages_nodemask+0x2e2/0x330 [91662.209780] kmalloc_order+0x28/0x90 [91662.210456] kmalloc_order_trace+0x1d/0xb0 [91662.211208] __kmalloc+0x203/0x250 [91662.211867] virtqueue_add+0x493/0xc70 [91662.212574] ? finish_wait+0x80/0x80 [91662.213338] virtqueue_add_sgs+0x80/0xa0 [91662.214300] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91662.215256] ? scsi_alloc_sgtables+0x84/0x1c0 [91662.216095] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91662.216998] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91662.218008] scsi_queue_rq+0x512/0xb10 [91662.218725] blk_mq_dispatch_rq_list+0x154/0x880 [91662.219567] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91662.220394] __blk_mq_sched_dispatch_requests+0x110/0x140 [91662.221334] blk_mq_sched_dispatch_requests+0x30/0x60 [91662.222234] __blk_mq_run_hw_queue+0x2b/0x70 [91662.223025] __blk_mq_delay_run_hw_queue+0x141/0x160 [91662.223913] blk_mq_sched_insert_requests+0x6c/0xf0 [91662.224773] blk_mq_flush_plug_list+0x135/0x220 [91662.225602] blk_flush_plug_list+0xd7/0x100 [91662.226373] blk_finish_plug+0x25/0x36 [91662.227082] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91662.228090] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91662.229039] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.230136] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.231199] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91662.232141] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91662.233016] ofd_commitrw+0x5b4/0xd20 [ofd] [91662.233784] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.234716] obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.235618] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91662.236533] ? flush_work+0x42/0x1d0 [91662.237216] ? internal_add_timer+0x42/0x70 [91662.237991] ? _cond_resched+0x15/0x30 [91662.238702] ? mutex_lock+0xe/0x30 [91662.239362] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91662.240330] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91662.241443] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91662.242581] ? finish_wait+0x80/0x80 [91662.243279] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91662.244158] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91662.245131] kthread+0x134/0x150 [91662.245752] ? set_kthread_struct+0x50/0x50 [91662.246518] ret_from_fork+0x1f/0x40 [91662.247200] CPU: 18 PID: 527679 Comm: ll_ost_io04_042 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91662.249857] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91662.252036] Call Trace: [91662.252886] dump_stack+0x41/0x60 [91662.253876] warn_alloc.cold.127+0x7b/0x108 [91662.255024] __alloc_pages_slowpath+0xcb2/0xcd0 [91662.256258] ? __alloc_pages_nodemask+0x166/0x330 [91662.257542] __alloc_pages_nodemask+0x2e2/0x330 [91662.258766] kmalloc_order+0x28/0x90 [91662.259815] kmalloc_order_trace+0x1d/0xb0 [91662.260937] __kmalloc+0x203/0x250 [91662.261995] virtqueue_add+0x493/0xc70 [91662.263029] virtqueue_add_sgs+0x80/0xa0 [91662.264149] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91662.265538] ? scsi_alloc_sgtables+0x84/0x1c0 [91662.266720] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91662.268007] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91662.269408] scsi_queue_rq+0x512/0xb10 [91662.270433] __blk_mq_try_issue_directly+0x163/0x200 [91662.271712] blk_mq_request_issue_directly+0x4e/0xb0 [91662.272769] blk_mq_try_issue_list_directly+0x62/0x100 [91662.273957] blk_mq_sched_insert_requests+0xa4/0xf0 [91662.275217] blk_mq_flush_plug_list+0x135/0x220 [91662.276396] blk_flush_plug_list+0xd7/0x100 [91662.277533] blk_finish_plug+0x25/0x36 [91662.278578] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91662.280055] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91662.281406] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.283041] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91662.284608] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91662.285971] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91662.287291] ofd_commitrw+0x5b4/0xd20 [ofd] [91662.288454] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.289844] obd_commitrw+0x1b6/0x370 [ptlrpc] [91662.291174] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91662.292668] ? flush_work+0x42/0x1d0 [91662.293674] ? internal_add_timer+0x42/0x70 [91662.294807] ? _cond_resched+0x15/0x30 [91662.295856] ? mutex_lock+0xe/0x30 [91662.296873] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91662.298350] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91662.299923] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91662.301492] ? ptlrpc_server_handle_request+0x4ee/0xbe0 [ptlrpc] [91662.303110] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91662.304427] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91662.305856] kthread+0x134/0x150 [91662.306831] ? set_kthread_struct+0x50/0x50 [91662.308061] ret_from_fork+0x1f/0x40 [91662.309057] warn_alloc_show_mem: 8 callbacks suppressed [91662.309059] Mem-Info: [91662.310578] active_anon:27195 inactive_anon:348146 isolated_anon:0 active_file:31289065 inactive_file:2676470 isolated_file:7 unevictable:0 dirty:3470 writeback:241 slab_reclaimable:773650 slab_unreclaimable:596524 mapped:24041 shmem:163 pagetables:5503 bounce:0 free:2303998 free_pcp:356 free_cma:0 [91662.316521] Node 0 active_anon:108780kB inactive_anon:1392584kB active_file:125163036kB inactive_file:10703304kB unevictable:0kB isolated(anon):0kB isolated(file):28kB mapped:96164kB dirty:13880kB writeback:964kB shmem:652kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35168kB pagetables:22012kB all_unreclaimable? no [91662.322146] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91662.327123] lowmem_reserve[]: 0 913 150007 150007 150007 [91662.328482] Node 0 DMA32 free:596676kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:136kB inactive_file:4kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91662.333165] lowmem_reserve[]: 0 0 149093 149093 149093 [91662.334109] Node 0 Normal free:8582736kB min:49268kB low:201940kB high:354612kB active_anon:108400kB inactive_anon:1388224kB active_file:125187432kB inactive_file:10700688kB unevictable:0kB writepending:14844kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1156kB local_pcp:92kB free_cma:0kB [91662.338501] lowmem_reserve[]: 0 0 0 0 0 [91662.339273] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [91662.341134] Node 0 DMA32: 169*4kB (UMH) 128*8kB (UMH) 198*16kB (UMEH) 118*32kB (UME) 165*64kB (UMEH) 157*128kB (UMEH) 134*256kB (UMEH) 113*512kB (UM) 88*1024kB (UMEH) 53*2048kB (UM) 65*4096kB (UM) = 596356kB [91662.344155] Node 0 Normal: 790249*4kB (UME) 600798*8kB (UME) 38466*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 8582836kB [91662.346360] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [91662.347825] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [91662.349252] 33971961 total pagecache pages [91662.350077] 2157 pages in swap cache [91662.350806] Swap cache stats: add 1309010, delete 1306870, find 55572/66885 [91662.352033] Free swap = 6294232kB [91662.352753] Total swap = 11075580kB [91662.353476] 39321462 pages RAM [91662.354150] 0 pages HighMem/MovableOnly [91662.354957] 905848 pages reserved [91662.355692] 0 pages hwpoisoned [91666.362101] warn_alloc: 197 callbacks suppressed [91666.362105] ll_ost_io04_072: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.364832] ll_ost_io02_038: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [91666.366660] CPU: 19 PID: 528060 Comm: ll_ost_io04_072 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.366663] / [91666.366666] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.369376] ,mems_allowed=0 [91666.369839] ll_ost_io00_077: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.370781] ll_ost_io00_029: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.372445] Call Trace: [91666.375649] dump_stack+0x41/0x60 [91666.383511] warn_alloc.cold.127+0x7b/0x108 [91666.384433] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.385417] ? __alloc_pages_nodemask+0x166/0x330 [91666.386452] __alloc_pages_nodemask+0x2e2/0x330 [91666.387393] kmalloc_order+0x28/0x90 [91666.388477] kmalloc_order_trace+0x1d/0xb0 [91666.389744] __kmalloc+0x203/0x250 [91666.390793] virtqueue_add+0x493/0xc70 [91666.391849] ? finish_wait+0x80/0x80 [91666.392621] virtqueue_add_sgs+0x80/0xa0 [91666.393456] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.394764] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.395971] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.397315] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.398579] scsi_queue_rq+0x512/0xb10 [91666.399386] blk_mq_dispatch_rq_list+0x154/0x880 [91666.400370] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91666.401440] __blk_mq_sched_dispatch_requests+0x110/0x140 [91666.402593] blk_mq_sched_dispatch_requests+0x30/0x60 [91666.403562] __blk_mq_run_hw_queue+0x2b/0x70 [91666.404498] __blk_mq_delay_run_hw_queue+0x141/0x160 [91666.405757] blk_mq_sched_insert_requests+0x6c/0xf0 [91666.406847] blk_mq_flush_plug_list+0x135/0x220 [91666.407909] blk_flush_plug_list+0xd7/0x100 [91666.409032] blk_finish_plug+0x25/0x36 [91666.409858] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.411079] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.412374] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.413913] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.415406] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.416572] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.417753] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.418681] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.419871] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.420918] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.422102] ? flush_work+0x42/0x1d0 [91666.423057] ? internal_add_timer+0x42/0x70 [91666.424142] ? _cond_resched+0x15/0x30 [91666.425151] ? mutex_lock+0xe/0x30 [91666.426072] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.427421] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.428008] ll_ost_io04_044: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91666.428013] ll_ost_io04_052: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.428035] ll_ost_io04_021: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.428781] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.430751] ,cpuset= [91666.433075] ? finish_wait+0x80/0x80 [91666.433079] ll_ost_io03_061: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.435428] / [91666.436695] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.437320] ,mems_allowed=0 [91666.438207] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.441044] kthread+0x134/0x150 [91666.445080] ? set_kthread_struct+0x50/0x50 [91666.445990] ret_from_fork+0x1f/0x40 [91666.446795] Mem-Info: [91666.446796] CPU: 16 PID: 527681 Comm: ll_ost_io04_044 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.447385] active_anon:27195 inactive_anon:348209 isolated_anon:0 active_file:30988674 inactive_file:2673866 isolated_file:0 unevictable:0 dirty:891 writeback:90 slab_reclaimable:775367 slab_unreclaimable:603608 mapped:24090 shmem:163 pagetables:5503 bounce:0 free:2598491 free_pcp:431 free_cma:0 [91666.449757] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.456542] Node 0 active_anon:108780kB inactive_anon:1392836kB active_file:123970892kB inactive_file:10695464kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:96360kB dirty:3564kB writeback:360kB shmem:652kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35168kB pagetables:22012kB all_unreclaimable? no [91666.458319] Call Trace: [91666.460807] ll_ost_io03_056: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.460851] ll_ost_io03_057: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91666.464031] Node 0 [91666.464723] dump_stack+0x41/0x60 [91666.467047] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91666.469349] warn_alloc.cold.127+0x7b/0x108 [91666.470025] lowmem_reserve[]: [91666.470854] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.474981] 0 [91666.475962] __alloc_pages_nodemask+0x2e2/0x330 [91666.476747] 913 [91666.477808] kmalloc_order+0x28/0x90 [91666.478385] 150007 [91666.479426] kmalloc_order_trace+0x1d/0xb0 [91666.480019] 150007 [91666.480910] __kmalloc+0x203/0x250 [91666.481567] 150007 [91666.482532] virtqueue_add+0x493/0xc70 [91666.484046] ? finish_wait+0x80/0x80 [91666.484725] Node 0 [91666.485693] virtqueue_add_sgs+0x80/0xa0 [91666.486294] DMA32 free:596580kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:136kB inactive_file:4kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91666.487215] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.487888] lowmem_reserve[]: [91666.488847] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.493264] 0 [91666.494490] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.495336] 0 [91666.496375] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.496975] 149093 [91666.498122] scsi_queue_rq+0x512/0xb10 [91666.498721] 149093 [91666.499959] blk_mq_dispatch_rq_list+0x154/0x880 [91666.500639] 149093 [91666.501562] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91666.503318] __blk_mq_sched_dispatch_requests+0x110/0x140 [91666.504006] Node 0 [91666.505054] blk_mq_sched_dispatch_requests+0x30/0x60 [91666.505641] Normal free:9755708kB min:49268kB low:201940kB high:354612kB active_anon:108400kB inactive_anon:1388476kB active_file:123996212kB inactive_file:10685088kB unevictable:0kB writepending:3656kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1644kB local_pcp:12kB free_cma:0kB [91666.506829] __blk_mq_run_hw_queue+0x2b/0x70 [91666.507489] lowmem_reserve[]: [91666.508613] __blk_mq_delay_run_hw_queue+0x141/0x160 [91666.513525] 0 [91666.514523] blk_mq_sched_insert_requests+0x6c/0xf0 [91666.515346] 0 [91666.517565] blk_mq_flush_plug_list+0x135/0x220 [91666.518154] 0 [91666.519256] blk_flush_plug_list+0xd7/0x100 [91666.519833] 0 [91666.520864] blk_finish_plug+0x25/0x36 [91666.521422] 0 [91666.522390] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.523877] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.524427] Node 0 [91666.525658] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.526216] DMA: [91666.527527] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.528160] 0*4kB [91666.529435] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.530027] 0*8kB [91666.531166] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.531784] 0*16kB [91666.532858] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.533444] 0*32kB [91666.534404] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.535005] 0*64kB [91666.536044] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.536644] 0*128kB [91666.537666] ? flush_work+0x42/0x1d0 [91666.538322] 0*256kB [91666.539365] ? internal_add_timer+0x42/0x70 [91666.539999] 0*512kB [91666.540848] ? _cond_resched+0x15/0x30 [91666.541485] 1*1024kB [91666.542438] ? mutex_lock+0xe/0x30 [91666.543070] (U) [91666.543978] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.544599] 1*2048kB [91666.545430] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.545976] (M) [91666.547076] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.547686] 2*4096kB [91666.548905] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.549445] (M) [91666.550670] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.551287] = 11264kB [91666.552276] kthread+0x134/0x150 [91666.552872] Node 0 [91666.553964] ? set_kthread_struct+0x50/0x50 [91666.554609] DMA32: [91666.555400] ret_from_fork+0x1f/0x40 [91666.556021] 169*4kB [91666.556971] CPU: 15 PID: 527441 Comm: ll_ost_io03_056 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.557515] (UMH) [91666.558409] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.559048] 128*8kB [91666.561426] Call Trace: [91666.562050] (UMH) [91666.563804] dump_stack+0x41/0x60 [91666.564441] 198*16kB [91666.565140] warn_alloc.cold.127+0x7b/0x108 [91666.565749] (UMEH) [91666.566571] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.567217] 127*32kB [91666.568188] ? do_get_write_access+0x37c/0x410 [jbd2] [91666.568792] (UMEH) [91666.569809] __alloc_pages_nodemask+0x2e2/0x330 [91666.570451] 165*64kB [91666.571559] kmalloc_order+0x28/0x90 [91666.572183] (UMEH) [91666.573190] kmalloc_order_trace+0x1d/0xb0 [91666.573830] 157*128kB [91666.574683] __kmalloc+0x203/0x250 [91666.575296] (UMEH) [91666.576226] virtqueue_add+0x493/0xc70 [91666.576891] 134*256kB [91666.577715] ? finish_wait+0x80/0x80 [91666.578329] (UMEH) [91666.579206] virtqueue_add_sgs+0x80/0xa0 [91666.579878] 113*512kB [91666.580729] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.581337] (UM) [91666.582239] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.582884] 88*1024kB [91666.584016] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.584606] (UMEH) [91666.585591] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.586252] 53*2048kB [91666.587341] scsi_queue_rq+0x512/0xb10 [91666.587960] (UM) [91666.589176] __blk_mq_try_issue_directly+0x163/0x200 [91666.589852] 65*4096kB [91666.590748] blk_mq_request_issue_directly+0x4e/0xb0 [91666.591358] (UM) [91666.592445] blk_mq_try_issue_list_directly+0x62/0x100 [91666.593121] = 596644kB [91666.594205] blk_mq_sched_insert_requests+0xa4/0xf0 [91666.594813] Node 0 [91666.595929] blk_mq_flush_plug_list+0x135/0x220 [91666.596601] Normal: [91666.597676] blk_flush_plug_list+0xd7/0x100 [91666.598303] 815967*4kB [91666.599313] blk_finish_plug+0x25/0x36 [91666.599960] (UME) [91666.600921] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.601599] 703885*8kB [91666.602483] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.603068] (UME) [91666.604274] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.604932] 49164*16kB [91666.606039] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.606644] (UMEH) [91666.607945] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.608602] 11*32kB [91666.609871] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.610482] (UH) [91666.611612] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.612226] 0*64kB [91666.613273] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.613819] 0*128kB [91666.614769] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.615340] 0*256kB [91666.616355] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.616941] 0*512kB [91666.617944] ? flush_work+0x42/0x1d0 [91666.618565] 0*1024kB [91666.619597] ? internal_add_timer+0x42/0x70 [91666.620248] 0*2048kB [91666.621108] ? _cond_resched+0x15/0x30 [91666.621753] 0*4096kB [91666.622712] ? mutex_lock+0xe/0x30 [91666.623353] = 9681924kB [91666.624232] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.624845] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [91666.625678] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.626311] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [91666.627412] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.629045] 33693975 total pagecache pages [91666.630264] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.631858] 2187 pages in swap cache [91666.632869] kthread+0x134/0x150 [91666.633814] Swap cache stats: add 1309075, delete 1306905, find 55606/66935 [91666.634915] ? set_kthread_struct+0x50/0x50 [91666.635767] Free swap = 6294488kB [91666.636570] ret_from_fork+0x1f/0x40 [91666.637967] Total swap = 11075580kB [91666.638925] CPU: 18 PID: 527236 Comm: ll_ost_io04_021 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.639710] 39321462 pages RAM [91666.640533] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.641319] 0 pages HighMem/MovableOnly [91666.643655] Call Trace: [91666.644369] 905848 pages reserved [91666.646074] dump_stack+0x41/0x60 [91666.646922] 0 pages hwpoisoned [91666.647555] warn_alloc.cold.127+0x7b/0x108 [91666.651810] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.653041] __alloc_pages_nodemask+0x2e2/0x330 [91666.654227] kmalloc_order+0x28/0x90 [91666.655237] kmalloc_order_trace+0x1d/0xb0 [91666.656332] __kmalloc+0x203/0x250 [91666.657160] virtqueue_add+0x493/0xc70 [91666.657935] virtqueue_add_sgs+0x80/0xa0 [91666.658800] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.660092] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.661314] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.662640] scsi_queue_rq+0x512/0xb10 [91666.663623] blk_mq_dispatch_rq_list+0x154/0x880 [91666.664766] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91666.665891] __blk_mq_sched_dispatch_requests+0x110/0x140 [91666.667215] blk_mq_sched_dispatch_requests+0x30/0x60 [91666.668423] __blk_mq_run_hw_queue+0x2b/0x70 [91666.669492] __blk_mq_delay_run_hw_queue+0x141/0x160 [91666.670694] blk_mq_sched_insert_requests+0x6c/0xf0 [91666.671870] blk_mq_flush_plug_list+0x135/0x220 [91666.672997] blk_flush_plug_list+0xd7/0x100 [91666.674063] blk_finish_plug+0x25/0x36 [91666.675072] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.676510] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.677811] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.679328] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.680804] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.682109] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.683310] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.684399] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.685688] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.686941] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.688219] ? flush_work+0x42/0x1d0 [91666.689151] ? internal_add_timer+0x42/0x70 [91666.690213] ? _cond_resched+0x15/0x30 [91666.691183] ? mutex_lock+0xe/0x30 [91666.692084] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.693420] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.694891] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.696382] ? lu_context_exit+0x2b/0x70 [obdclass] [91666.697637] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.698842] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.700165] kthread+0x134/0x150 [91666.701014] ? set_kthread_struct+0x50/0x50 [91666.702042] ret_from_fork+0x1f/0x40 [91666.702958] CPU: 2 PID: 527259 Comm: ll_ost_io00_029 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.705337] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.707399] Call Trace: [91666.708178] dump_stack+0x41/0x60 [91666.709115] warn_alloc.cold.127+0x7b/0x108 [91666.710209] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.711372] ? __alloc_pages_nodemask+0x166/0x330 [91666.712560] __alloc_pages_nodemask+0x2e2/0x330 [91666.713717] kmalloc_order+0x28/0x90 [91666.714728] kmalloc_order_trace+0x1d/0xb0 [91666.715826] __kmalloc+0x203/0x250 [91666.716776] virtqueue_add+0x493/0xc70 [91666.717791] ? finish_wait+0x80/0x80 [91666.718764] virtqueue_add_sgs+0x80/0xa0 [91666.719830] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.721180] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.722325] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.723632] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.725023] scsi_queue_rq+0x512/0xb10 [91666.726062] __blk_mq_try_issue_directly+0x163/0x200 [91666.727359] blk_mq_request_issue_directly+0x4e/0xb0 [91666.728632] blk_mq_try_issue_list_directly+0x62/0x100 [91666.729943] blk_mq_sched_insert_requests+0xa4/0xf0 [91666.731174] blk_mq_flush_plug_list+0x135/0x220 [91666.732320] blk_flush_plug_list+0xd7/0x100 [91666.733407] blk_finish_plug+0x25/0x36 [91666.734383] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.735819] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.737151] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.738688] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.740177] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.741497] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.742726] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.743808] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.745104] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.746361] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.747663] ? flush_work+0x42/0x1d0 [91666.748617] ? internal_add_timer+0x42/0x70 [91666.749711] ? _cond_resched+0x15/0x30 [91666.750703] ? mutex_lock+0xe/0x30 [91666.751604] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.752991] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.754473] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.755970] ? lu_context_exit+0x12/0x70 [obdclass] [91666.757248] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.758485] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.759828] kthread+0x134/0x150 [91666.760706] ? set_kthread_struct+0x50/0x50 [91666.761775] ret_from_fork+0x1f/0x40 [91666.762741] CPU: 17 PID: 527689 Comm: ll_ost_io04_052 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.765013] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.766689] Call Trace: [91666.767280] dump_stack+0x41/0x60 [91666.768001] warn_alloc.cold.127+0x7b/0x108 [91666.768832] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.769704] __alloc_pages_nodemask+0x2e2/0x330 [91666.770581] kmalloc_order+0x28/0x90 [91666.771317] kmalloc_order_trace+0x1d/0xb0 [91666.772119] __kmalloc+0x203/0x250 [91666.772812] virtqueue_add+0x493/0xc70 [91666.773570] ? finish_wait+0x80/0x80 [91666.774288] virtqueue_add_sgs+0x80/0xa0 [91666.775044] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.776036] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.776851] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.777770] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.778775] scsi_queue_rq+0x512/0xb10 [91666.779593] __blk_mq_try_issue_directly+0x163/0x200 [91666.780625] blk_mq_request_issue_directly+0x4e/0xb0 [91666.781695] blk_mq_try_issue_list_directly+0x62/0x100 [91666.782754] blk_mq_sched_insert_requests+0xa4/0xf0 [91666.783781] blk_mq_flush_plug_list+0x135/0x220 [91666.784725] blk_flush_plug_list+0xd7/0x100 [91666.785608] blk_finish_plug+0x25/0x36 [91666.786410] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.787572] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.788623] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.789765] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.790900] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.791982] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.792915] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.793735] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.794687] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.795644] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.796616] ? flush_work+0x42/0x1d0 [91666.797350] ? internal_add_timer+0x42/0x70 [91666.798149] ? _cond_resched+0x15/0x30 [91666.798910] ? mutex_lock+0xe/0x30 [91666.799617] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.800659] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.801800] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.802959] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.803876] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.804899] kthread+0x134/0x150 [91666.805604] ? set_kthread_struct+0x50/0x50 [91666.806634] ret_from_fork+0x1f/0x40 [91666.807343] CPU: 13 PID: 527442 Comm: ll_ost_io03_057 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.810663] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.812940] Call Trace: [91666.813769] dump_stack+0x41/0x60 [91666.814759] warn_alloc.cold.127+0x7b/0x108 [91666.815909] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.817147] __alloc_pages_nodemask+0x2e2/0x330 [91666.818352] kmalloc_order+0x28/0x90 [91666.819408] kmalloc_order_trace+0x1d/0xb0 [91666.820586] __kmalloc+0x203/0x250 [91666.821629] ? __find_get_block+0x23a/0x2b0 [91666.822781] virtqueue_add+0x493/0xc70 [91666.823864] ? finish_wait+0x80/0x80 [91666.824927] virtqueue_add_sgs+0x80/0xa0 [91666.826051] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.827418] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.828573] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.829813] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.831168] scsi_queue_rq+0x512/0xb10 [91666.832166] blk_mq_dispatch_rq_list+0x154/0x880 [91666.833312] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91666.834465] __blk_mq_sched_dispatch_requests+0x110/0x140 [91666.835759] blk_mq_sched_dispatch_requests+0x30/0x60 [91666.837009] __blk_mq_run_hw_queue+0x2b/0x70 [91666.838186] __blk_mq_delay_run_hw_queue+0x141/0x160 [91666.839437] blk_mq_sched_insert_requests+0x6c/0xf0 [91666.840698] blk_mq_flush_plug_list+0x135/0x220 [91666.841857] blk_flush_plug_list+0xd7/0x100 [91666.842923] blk_finish_plug+0x25/0x36 [91666.843937] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.845388] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.846625] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.848120] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.849597] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.850862] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.852096] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.853255] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.854557] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.855836] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.857173] ? flush_work+0x42/0x1d0 [91666.858171] ? internal_add_timer+0x42/0x70 [91666.859239] ? _cond_resched+0x15/0x30 [91666.860230] ? mutex_lock+0xe/0x30 [91666.861169] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.862546] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.864000] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.865438] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.866436] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.867584] kthread+0x134/0x150 [91666.868494] ? set_kthread_struct+0x50/0x50 [91666.869545] ret_from_fork+0x1f/0x40 [91666.870474] CPU: 14 PID: 527446 Comm: ll_ost_io03_061 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.873178] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.875235] Call Trace: [91666.875994] dump_stack+0x41/0x60 [91666.876895] warn_alloc.cold.127+0x7b/0x108 [91666.878004] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.879144] __alloc_pages_nodemask+0x2e2/0x330 [91666.880272] kmalloc_order+0x28/0x90 [91666.881149] kmalloc_order_trace+0x1d/0xb0 [91666.882162] __kmalloc+0x203/0x250 [91666.883078] ? __wake_up_common+0x7a/0x190 [91666.884134] virtqueue_add+0x493/0xc70 [91666.885123] ? finish_wait+0x80/0x80 [91666.886077] virtqueue_add_sgs+0x80/0xa0 [91666.887083] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.888397] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.889497] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.890728] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.892062] scsi_queue_rq+0x512/0xb10 [91666.893040] __blk_mq_try_issue_directly+0x163/0x200 [91666.894229] blk_mq_request_issue_directly+0x4e/0xb0 [91666.895320] blk_mq_try_issue_list_directly+0x62/0x100 [91666.896559] blk_mq_sched_insert_requests+0xa4/0xf0 [91666.897750] blk_mq_flush_plug_list+0x135/0x220 [91666.898866] blk_flush_plug_list+0xd7/0x100 [91666.899904] blk_finish_plug+0x25/0x36 [91666.900871] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.902267] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.903284] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.904630] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.906074] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.907055] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.907967] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.908785] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.909766] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.910696] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.911693] ? flush_work+0x42/0x1d0 [91666.912422] ? internal_add_timer+0x42/0x70 [91666.913246] ? _cond_resched+0x15/0x30 [91666.913994] ? mutex_lock+0xe/0x30 [91666.914671] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.915681] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.916815] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.917992] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.918914] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.919930] kthread+0x134/0x150 [91666.920606] ? set_kthread_struct+0x50/0x50 [91666.921554] ret_from_fork+0x1f/0x40 [91666.922288] CPU: 10 PID: 527243 Comm: ll_ost_io02_038 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.924763] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.926847] Call Trace: [91666.927444] dump_stack+0x41/0x60 [91666.928101] warn_alloc.cold.127+0x7b/0x108 [91666.928882] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.929702] ? __alloc_pages_nodemask+0x166/0x330 [91666.930554] __alloc_pages_nodemask+0x2e2/0x330 [91666.931490] kmalloc_order+0x28/0x90 [91666.932215] kmalloc_order_trace+0x1d/0xb0 [91666.932999] __kmalloc+0x203/0x250 [91666.933680] virtqueue_add+0x493/0xc70 [91666.934411] virtqueue_add_sgs+0x80/0xa0 [91666.935138] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.936077] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.936931] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.938032] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.939362] scsi_queue_rq+0x512/0xb10 [91666.940359] __blk_mq_try_issue_directly+0x163/0x200 [91666.941621] blk_mq_request_issue_directly+0x4e/0xb0 [91666.942889] blk_mq_try_issue_list_directly+0x62/0x100 [91666.944137] blk_mq_sched_insert_requests+0xa4/0xf0 [91666.945344] blk_mq_flush_plug_list+0x135/0x220 [91666.946391] blk_flush_plug_list+0xd7/0x100 [91666.947170] blk_finish_plug+0x25/0x36 [91666.947881] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91666.948956] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91666.949941] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.951208] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91666.952508] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91666.953600] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91666.954613] ofd_commitrw+0x5b4/0xd20 [ofd] [91666.955580] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.956862] obd_commitrw+0x1b6/0x370 [ptlrpc] [91666.958115] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91666.959416] ? flush_work+0x42/0x1d0 [91666.960379] ? internal_add_timer+0x42/0x70 [91666.961436] ? _cond_resched+0x15/0x30 [91666.962417] ? mutex_lock+0xe/0x30 [91666.963330] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91666.964669] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91666.966184] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91666.967732] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91666.968971] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91666.970299] kthread+0x134/0x150 [91666.971004] ? set_kthread_struct+0x50/0x50 [91666.972013] ret_from_fork+0x1f/0x40 [91666.972949] CPU: 0 PID: 527782 Comm: ll_ost_io00_077 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91666.975338] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91666.977351] Call Trace: [91666.978079] dump_stack+0x41/0x60 [91666.978991] warn_alloc.cold.127+0x7b/0x108 [91666.980071] __alloc_pages_slowpath+0xcb2/0xcd0 [91666.981213] ? __alloc_pages_nodemask+0x166/0x330 [91666.982390] __alloc_pages_nodemask+0x2e2/0x330 [91666.983530] kmalloc_order+0x28/0x90 [91666.984490] kmalloc_order_trace+0x1d/0xb0 [91666.985526] __kmalloc+0x203/0x250 [91666.986443] virtqueue_add+0x493/0xc70 [91666.987448] ? finish_wait+0x80/0x80 [91666.988422] virtqueue_add_sgs+0x80/0xa0 [91666.989435] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91666.990717] ? scsi_alloc_sgtables+0x84/0x1c0 [91666.991787] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91666.992999] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91666.994320] scsi_queue_rq+0x512/0xb10 [91666.995283] __blk_mq_try_issue_directly+0x163/0x200 [91666.996483] blk_mq_request_issue_directly+0x4e/0xb0 [91666.997692] blk_mq_try_issue_list_directly+0x62/0x100 [91666.998925] blk_mq_sched_insert_requests+0xa4/0xf0 [91667.000100] blk_mq_flush_plug_list+0x135/0x220 [91667.001250] blk_flush_plug_list+0xd7/0x100 [91667.002297] blk_finish_plug+0x25/0x36 [91667.003045] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91667.004097] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91667.005096] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91667.006244] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91667.007334] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91667.008311] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91667.009253] ofd_commitrw+0x5b4/0xd20 [ofd] [91667.010096] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91667.011153] obd_commitrw+0x1b6/0x370 [ptlrpc] [91667.012355] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91667.013607] ? flush_work+0x42/0x1d0 [91667.014531] ? internal_add_timer+0x42/0x70 [91667.015345] ? _cond_resched+0x15/0x30 [91667.016093] ? mutex_lock+0xe/0x30 [91667.016785] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91667.017864] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91667.019284] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91667.020762] ? lprocfs_counter_add+0x86/0x140 [obdclass] [91667.022119] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91667.023361] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91667.024691] kthread+0x134/0x150 [91667.025555] ? set_kthread_struct+0x50/0x50 [91667.026579] ret_from_fork+0x1f/0x40 [91674.735398] warn_alloc: 2 callbacks suppressed [91674.735401] ll_ost_io02_079: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91674.739369] CPU: 11 PID: 527872 Comm: ll_ost_io02_079 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91674.739852] ll_ost_io01_078: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91674.741788] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91674.741789] ,cpuset=/ [91674.743511] Call Trace: [91674.744614] ll_ost_io03_073: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [91674.745641] ,mems_allowed=0 [91674.745903] ll_ost_io02_006: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [91674.746441] dump_stack+0x41/0x60 [91674.747280] / [91674.747280] ,mems_allowed=0 [91674.748164] ll_ost_io03_033: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [91674.749512] warn_alloc.cold.127+0x7b/0x108 [91674.750443] / [91674.750443] ,mems_allowed=0 [91674.752130] ll_ost_io03_054: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [91674.752980] __alloc_pages_slowpath+0xcb2/0xcd0 [91674.753884] / [91674.753884] ,mems_allowed=0 [91674.754599] __alloc_pages_nodemask+0x2e2/0x330 [91674.755384] ,mems_allowed=0 [91674.756154] ll_ost_io01_073: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91674.756166] ll_ost_io00_063: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91674.757921] kmalloc_order+0x28/0x90 [91674.759638] kmalloc_order_trace+0x1d/0xb0 [91674.761106] ll_ost_io03_013: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [91674.762929] __kmalloc+0x203/0x250 [91674.764011] ,cpuset= [91674.764594] virtqueue_add+0x493/0xc70 [91674.765361] / [91674.766386] ? finish_wait+0x80/0x80 [91674.767149] ,mems_allowed=0 [91674.767997] ll_ost_io02_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [91674.770070] virtqueue_add_sgs+0x80/0xa0 [91674.773604] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91674.789437] ? scsi_alloc_sgtables+0x84/0x1c0 [91674.790656] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91674.792014] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91674.793249] scsi_queue_rq+0x512/0xb10 [91674.794222] __blk_mq_try_issue_directly+0x163/0x200 [91674.795337] blk_mq_request_issue_directly+0x4e/0xb0 [91674.796439] blk_mq_try_issue_list_directly+0x62/0x100 [91674.797808] blk_mq_sched_insert_requests+0xa4/0xf0 [91674.799142] blk_mq_flush_plug_list+0x135/0x220 [91674.800390] blk_flush_plug_list+0xd7/0x100 [91674.801549] blk_finish_plug+0x25/0x36 [91674.802616] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91674.804137] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91674.805573] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91674.807209] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91674.808782] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91674.810145] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91674.811410] ofd_commitrw+0x5b4/0xd20 [ofd] [91674.812510] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91674.813850] obd_commitrw+0x1b6/0x370 [ptlrpc] [91674.814837] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91674.815923] ? flush_work+0x42/0x1d0 [91674.816667] ? internal_add_timer+0x42/0x70 [91674.817476] ? _cond_resched+0x15/0x30 [91674.818202] ? mutex_lock+0xe/0x30 [91674.818859] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91674.819822] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91674.820875] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91674.822027] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91674.823087] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91674.824115] kthread+0x134/0x150 [91674.824785] ? set_kthread_struct+0x50/0x50 [91674.825608] ret_from_fork+0x1f/0x40 [91674.826349] warn_alloc_show_mem: 9 callbacks suppressed [91674.826349] CPU: 8 PID: 527702 Comm: ll_ost_io02_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91674.826352] Mem-Info: [91674.826353] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91674.827321] active_anon:27194 inactive_anon:348240 isolated_anon:0 active_file:31362709 inactive_file:2409420 isolated_file:32 unevictable:0 dirty:2359 writeback:0 slab_reclaimable:776291 slab_unreclaimable:598859 mapped:24227 shmem:163 pagetables:5459 bounce:0 free:2492419 free_pcp:601 free_cma:0 [91674.829728] Call Trace: [91674.830351] Node 0 active_anon:108776kB inactive_anon:1392960kB active_file:125450836kB inactive_file:9637680kB unevictable:0kB isolated(anon):0kB isolated(file):128kB mapped:96908kB dirty:9436kB writeback:0kB shmem:652kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35280kB pagetables:21836kB all_unreclaimable? no [91674.832121] dump_stack+0x41/0x60 [91674.840004] Node 0 [91674.840673] warn_alloc.cold.127+0x7b/0x108 [91674.846559] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [91674.847388] __alloc_pages_slowpath+0xcb2/0xcd0 [91674.848135] lowmem_reserve[]: [91674.849049] ? __alloc_pages_nodemask+0x166/0x330 [91674.854164] 0 [91674.855192] __alloc_pages_nodemask+0x2e2/0x330 [91674.856159] 913 [91674.857189] kmalloc_order+0x28/0x90 [91674.857868] 150007 [91674.858918] kmalloc_order_trace+0x1d/0xb0 [91674.859650] 150007 [91674.860533] __kmalloc+0x203/0x250 [91674.861319] 150007 [91674.862267] virtqueue_add+0x493/0xc70 [91674.863874] ? finish_wait+0x80/0x80 [91674.864655] Node 0 [91674.865568] virtqueue_add_sgs+0x80/0xa0 [91674.866245] DMA32 free:596528kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:144kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:752kB local_pcp:248kB free_cma:0kB [91674.867103] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91674.867766] lowmem_reserve[]: [91674.868667] ? scsi_alloc_sgtables+0x84/0x1c0 [91674.873980] 0 [91674.875171] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91674.876211] 0 [91674.877240] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91674.878010] 149093 [91674.879124] scsi_queue_rq+0x512/0xb10 [91674.879862] 149093 [91674.881061] __blk_mq_try_issue_directly+0x163/0x200 [91674.881899] 149093 [91674.882817] blk_mq_request_issue_directly+0x4e/0xb0 [91674.884729] blk_mq_try_issue_list_directly+0x62/0x100 [91674.885572] Node 0 [91674.886674] blk_mq_sched_insert_requests+0xa4/0xf0 [91674.887373] Normal free:9342712kB min:49268kB low:201940kB high:354612kB active_anon:108396kB inactive_anon:1388316kB active_file:125471728kB inactive_file:9637504kB unevictable:0kB writepending:9436kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1520kB local_pcp:36kB free_cma:0kB [91674.888474] blk_mq_flush_plug_list+0x135/0x220 [91674.889275] lowmem_reserve[]: [91674.890342] blk_flush_plug_list+0xd7/0x100 [91674.896113] 0 [91674.897137] blk_finish_plug+0x25/0x36 [91674.898136] 0 [91674.899118] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91674.899840] 0 [91674.900734] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91674.901419] 0 [91674.902628] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91674.903315] 0 [91674.904436] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91674.906430] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91674.907098] Node 0 [91674.908389] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91674.909056] DMA: [91674.910189] ofd_commitrw+0x5b4/0xd20 [ofd] [91674.910991] 0*4kB [91674.912042] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91674.912793] 0*8kB [91674.913759] obd_commitrw+0x1b6/0x370 [ptlrpc] [91674.914518] 0*16kB [91674.915526] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91674.916266] 0*32kB [91674.917271] ? flush_work+0x42/0x1d0 [91674.918064] 0*64kB [91674.919112] ? internal_add_timer+0x42/0x70 [91674.919774] 0*128kB [91674.920643] ? _cond_resched+0x15/0x30 [91674.921439] 0*256kB [91674.922403] ? mutex_lock+0xe/0x30 [91674.923220] 0*512kB [91674.924143] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91674.924775] 1*1024kB [91674.925619] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91674.926362] (U) [91674.927482] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91674.928269] 1*2048kB [91674.929498] ? lu_context_exit+0x41/0x70 [obdclass] [91674.930192] (M) [91674.931428] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91674.932201] 2*4096kB [91674.933276] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91674.933967] (M) [91674.934974] kthread+0x134/0x150 [91674.935597] = 11264kB [91674.936689] ? set_kthread_struct+0x50/0x50 [91674.937259] Node 0 [91674.938067] ret_from_fork+0x1f/0x40 [91674.938844] DMA32: [91674.939813] CPU: 12 PID: 527265 Comm: ll_ost_io03_033 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91674.940480] 54*4kB [91674.941299] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91674.942044] (UMH) [91674.944346] Call Trace: [91674.945114] 91*8kB [91674.946821] dump_stack+0x41/0x60 [91674.947602] (UMH) [91674.948263] warn_alloc.cold.127+0x7b/0x108 [91674.949018] 198*16kB [91674.949788] __alloc_pages_slowpath+0xcb2/0xcd0 [91674.950528] (UMEH) [91674.951439] ? __alloc_pages_nodemask+0x166/0x330 [91674.952252] 116*32kB [91674.953201] __alloc_pages_nodemask+0x2e2/0x330 [91674.953962] (UMEH) [91674.954940] kmalloc_order+0x28/0x90 [91674.955784] 167*64kB [91674.956757] kmalloc_order_trace+0x1d/0xb0 [91674.957534] (UMEH) [91674.958356] __kmalloc+0x203/0x250 [91674.959170] 158*128kB [91674.960088] virtqueue_add+0x493/0xc70 [91674.960873] (UME) [91674.961658] ? finish_wait+0x80/0x80 [91674.962509] 133*256kB [91674.963337] virtqueue_add_sgs+0x80/0xa0 [91674.964115] (UMEH) [91674.964922] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91674.965748] 113*512kB [91674.966635] ? scsi_alloc_sgtables+0x84/0x1c0 [91674.967386] (UM) [91674.968495] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91674.969315] 87*1024kB [91674.970278] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91674.971010] (UMEH) [91674.972077] scsi_queue_rq+0x512/0xb10 [91674.972905] 52*2048kB [91674.974094] __blk_mq_try_issue_directly+0x163/0x200 [91674.974868] (UM) [91674.975735] blk_mq_request_issue_directly+0x4e/0xb0 [91674.976562] 66*4096kB [91674.977601] blk_mq_try_issue_list_directly+0x62/0x100 [91674.978347] (UM) [91674.979395] blk_mq_sched_insert_requests+0xa4/0xf0 [91674.980244] = 596560kB [91674.981317] blk_mq_flush_plug_list+0x135/0x220 [91674.982072] Node 0 [91674.983099] blk_flush_plug_list+0xd7/0x100 [91674.983938] Normal: [91674.984919] blk_finish_plug+0x25/0x36 [91674.985732] 741251*4kB [91674.986641] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91674.987434] (UMEH) [91674.988275] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91674.989100] 681320*8kB [91674.990252] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91674.990974] (UEH) [91674.992054] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91674.992872] 55668*16kB [91674.994149] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91674.994886] (UEH) [91674.996126] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91674.996959] 10*32kB [91674.998056] ofd_commitrw+0x5b4/0xd20 [ofd] [91674.998799] (EH) [91674.999807] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.000536] 0*64kB [91675.001439] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.002134] 0*128kB [91675.003125] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.003845] 0*256kB [91675.004793] ? flush_work+0x42/0x1d0 [91675.005568] 0*512kB [91675.006547] ? internal_add_timer+0x42/0x70 [91675.007340] 0*1024kB [91675.008163] ? _cond_resched+0x15/0x30 [91675.008922] 0*2048kB [91675.009851] ? mutex_lock+0xe/0x30 [91675.010644] 0*4096kB [91675.011481] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.012253] = 9306572kB [91675.013057] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.013826] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [91675.014874] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.015677] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [91675.016859] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.018971] 33789203 total pagecache pages [91675.020183] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.022155] 2196 pages in swap cache [91675.023131] kthread+0x134/0x150 [91675.024323] Swap cache stats: add 1309187, delete 1307008, find 55708/67058 [91675.025391] ? set_kthread_struct+0x50/0x50 [91675.026468] Free swap = 6294744kB [91675.027228] ret_from_fork+0x1f/0x40 [91675.028978] Total swap = 11075580kB [91675.029902] CPU: 15 PID: 527033 Comm: ll_ost_io03_013 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91675.030855] 39321462 pages RAM [91675.031803] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91675.032890] 0 pages HighMem/MovableOnly [91675.035316] Call Trace: [91675.036315] 905848 pages reserved [91675.038061] dump_stack+0x41/0x60 [91675.039263] 0 pages hwpoisoned [91675.039921] warn_alloc.cold.127+0x7b/0x108 [91675.043526] __alloc_pages_slowpath+0xcb2/0xcd0 [91675.044507] ? __alloc_pages_nodemask+0x166/0x330 [91675.045520] __alloc_pages_nodemask+0x2e2/0x330 [91675.046509] kmalloc_order+0x28/0x90 [91675.047335] kmalloc_order_trace+0x1d/0xb0 [91675.048239] __kmalloc+0x203/0x250 [91675.049032] virtqueue_add+0x493/0xc70 [91675.049879] ? finish_wait+0x80/0x80 [91675.050702] virtqueue_add_sgs+0x80/0xa0 [91675.051587] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91675.052706] ? scsi_alloc_sgtables+0x84/0x1c0 [91675.053655] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91675.054726] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91675.055898] scsi_queue_rq+0x512/0xb10 [91675.056764] __blk_mq_try_issue_directly+0x163/0x200 [91675.057828] blk_mq_request_issue_directly+0x4e/0xb0 [91675.058877] blk_mq_try_issue_list_directly+0x62/0x100 [91675.059981] blk_mq_sched_insert_requests+0xa4/0xf0 [91675.061032] blk_mq_flush_plug_list+0x135/0x220 [91675.062005] blk_flush_plug_list+0xd7/0x100 [91675.062925] blk_finish_plug+0x25/0x36 [91675.063779] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91675.065009] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91675.066141] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.067445] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.068754] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91675.069873] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91675.070919] ofd_commitrw+0x5b4/0xd20 [ofd] [91675.071866] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.073011] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.074117] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.075241] ? flush_work+0x42/0x1d0 [91675.076073] ? internal_add_timer+0x42/0x70 [91675.076981] ? _cond_resched+0x15/0x30 [91675.077813] ? mutex_lock+0xe/0x30 [91675.078604] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.079763] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.081022] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.082290] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.083344] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.084494] kthread+0x134/0x150 [91675.085237] ? set_kthread_struct+0x50/0x50 [91675.086127] ret_from_fork+0x1f/0x40 [91675.086936] CPU: 4 PID: 527342 Comm: ll_ost_io01_073 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91675.089022] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91675.090614] Call Trace: [91675.091360] dump_stack+0x41/0x60 [91675.092020] warn_alloc.cold.127+0x7b/0x108 [91675.092807] __alloc_pages_slowpath+0xcb2/0xcd0 [91675.093631] ? __alloc_pages_nodemask+0x166/0x330 [91675.094483] __alloc_pages_nodemask+0x2e2/0x330 [91675.095302] kmalloc_order+0x28/0x90 [91675.096003] kmalloc_order_trace+0x1d/0xb0 [91675.096774] __kmalloc+0x203/0x250 [91675.097436] virtqueue_add+0x493/0xc70 [91675.098164] ? finish_wait+0x80/0x80 [91675.098839] virtqueue_add_sgs+0x80/0xa0 [91675.099588] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91675.100524] ? scsi_alloc_sgtables+0x84/0x1c0 [91675.101325] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91675.102253] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91675.103274] scsi_queue_rq+0x512/0xb10 [91675.103982] __blk_mq_try_issue_directly+0x163/0x200 [91675.104877] blk_mq_request_issue_directly+0x4e/0xb0 [91675.105793] blk_mq_try_issue_list_directly+0x62/0x100 [91675.106715] blk_mq_sched_insert_requests+0xa4/0xf0 [91675.107608] blk_mq_flush_plug_list+0x135/0x220 [91675.108458] blk_flush_plug_list+0xd7/0x100 [91675.109256] blk_finish_plug+0x25/0x36 [91675.109973] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91675.111031] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91675.112005] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.113143] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.114249] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91675.115225] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91675.116144] ofd_commitrw+0x5b4/0xd20 [ofd] [91675.116944] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.117896] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.118831] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.119774] ? flush_work+0x42/0x1d0 [91675.120475] ? internal_add_timer+0x42/0x70 [91675.121274] ? _cond_resched+0x15/0x30 [91675.121989] ? mutex_lock+0xe/0x30 [91675.122664] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.123669] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.124767] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.126161] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.127177] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.128201] kthread+0x134/0x150 [91675.128859] ? set_kthread_struct+0x50/0x50 [91675.129637] ret_from_fork+0x1f/0x40 [91675.130335] CPU: 1 PID: 527756 Comm: ll_ost_io00_063 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91675.132954] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91675.134758] Call Trace: [91675.135525] dump_stack+0x41/0x60 [91675.136478] warn_alloc.cold.127+0x7b/0x108 [91675.137416] __alloc_pages_slowpath+0xcb2/0xcd0 [91675.138403] ? __alloc_pages_nodemask+0x166/0x330 [91675.139412] __alloc_pages_nodemask+0x2e2/0x330 [91675.140393] kmalloc_order+0x28/0x90 [91675.141207] kmalloc_order_trace+0x1d/0xb0 [91675.142116] __kmalloc+0x203/0x250 [91675.142898] virtqueue_add+0x493/0xc70 [91675.143738] virtqueue_add_sgs+0x80/0xa0 [91675.144607] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91675.145748] ? scsi_alloc_sgtables+0x84/0x1c0 [91675.146709] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91675.147767] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91675.148947] scsi_queue_rq+0x512/0xb10 [91675.149790] __blk_mq_try_issue_directly+0x163/0x200 [91675.150843] blk_mq_request_issue_directly+0x4e/0xb0 [91675.151906] blk_mq_try_issue_list_directly+0x62/0x100 [91675.153011] blk_mq_sched_insert_requests+0xa4/0xf0 [91675.154055] blk_mq_flush_plug_list+0x135/0x220 [91675.155069] blk_flush_plug_list+0xd7/0x100 [91675.156031] blk_finish_plug+0x25/0x36 [91675.156967] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91675.158309] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91675.159448] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.160774] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.163499] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91675.165334] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91675.166828] ofd_commitrw+0x5b4/0xd20 [ofd] [91675.168313] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.170122] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.171722] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.172924] ? flush_work+0x42/0x1d0 [91675.173757] ? internal_add_timer+0x42/0x70 [91675.174673] ? _cond_resched+0x15/0x30 [91675.175523] ? mutex_lock+0xe/0x30 [91675.176310] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.177480] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.178759] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.180032] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.181109] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.182247] kthread+0x134/0x150 [91675.182991] ? set_kthread_struct+0x50/0x50 [91675.183891] ret_from_fork+0x1f/0x40 [91675.184719] CPU: 14 PID: 527458 Comm: ll_ost_io03_073 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91675.187681] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91675.189831] Call Trace: [91675.190652] dump_stack+0x41/0x60 [91675.191624] warn_alloc.cold.127+0x7b/0x108 [91675.192767] __alloc_pages_slowpath+0xcb2/0xcd0 [91675.193960] ? __alloc_pages_nodemask+0x166/0x330 [91675.195181] __alloc_pages_nodemask+0x2e2/0x330 [91675.196357] kmalloc_order+0x28/0x90 [91675.197377] kmalloc_order_trace+0x1d/0xb0 [91675.198500] __kmalloc+0x203/0x250 [91675.199476] virtqueue_add+0x493/0xc70 [91675.200536] ? finish_wait+0x80/0x80 [91675.201563] virtqueue_add_sgs+0x80/0xa0 [91675.202669] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91675.204033] ? scsi_alloc_sgtables+0x84/0x1c0 [91675.205205] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91675.206481] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91675.207869] scsi_queue_rq+0x512/0xb10 [91675.208868] __blk_mq_try_issue_directly+0x163/0x200 [91675.210095] blk_mq_request_issue_directly+0x4e/0xb0 [91675.211337] blk_mq_try_issue_list_directly+0x62/0x100 [91675.212601] blk_mq_sched_insert_requests+0xa4/0xf0 [91675.213818] blk_mq_flush_plug_list+0x135/0x220 [91675.214979] blk_flush_plug_list+0xd7/0x100 [91675.216063] blk_finish_plug+0x25/0x36 [91675.217073] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91675.218530] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91675.219880] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.221446] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.222964] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91675.224285] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91675.225525] ofd_commitrw+0x5b4/0xd20 [ofd] [91675.226649] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.227972] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.229228] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.230558] ? flush_work+0x42/0x1d0 [91675.231533] ? internal_add_timer+0x42/0x70 [91675.232636] ? _cond_resched+0x15/0x30 [91675.233617] ? mutex_lock+0xe/0x30 [91675.234545] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.235925] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.237435] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.238984] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.240299] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.241740] kthread+0x134/0x150 [91675.242658] ? set_kthread_struct+0x50/0x50 [91675.243735] ret_from_fork+0x1f/0x40 [91675.244735] CPU: 7 PID: 528147 Comm: ll_ost_io01_078 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91675.248154] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91675.250365] Call Trace: [91675.251173] dump_stack+0x41/0x60 [91675.252137] warn_alloc.cold.127+0x7b/0x108 [91675.253277] __alloc_pages_slowpath+0xcb2/0xcd0 [91675.254508] __alloc_pages_nodemask+0x2e2/0x330 [91675.255736] kmalloc_order+0x28/0x90 [91675.256716] kmalloc_order_trace+0x1d/0xb0 [91675.257814] __kmalloc+0x203/0x250 [91675.258780] virtqueue_add+0x493/0xc70 [91675.259760] ? finish_wait+0x80/0x80 [91675.260717] virtqueue_add_sgs+0x80/0xa0 [91675.261778] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91675.263157] ? scsi_alloc_sgtables+0x84/0x1c0 [91675.264318] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91675.265597] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91675.267003] scsi_queue_rq+0x512/0xb10 [91675.268035] blk_mq_dispatch_rq_list+0x154/0x880 [91675.269244] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [91675.270419] __blk_mq_sched_dispatch_requests+0x110/0x140 [91675.271780] blk_mq_sched_dispatch_requests+0x30/0x60 [91675.273092] __blk_mq_run_hw_queue+0x2b/0x70 [91675.274250] __blk_mq_delay_run_hw_queue+0x141/0x160 [91675.275522] blk_mq_sched_insert_requests+0x6c/0xf0 [91675.276772] blk_mq_flush_plug_list+0x135/0x220 [91675.277990] blk_flush_plug_list+0xd7/0x100 [91675.279096] blk_finish_plug+0x25/0x36 [91675.280092] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91675.281513] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91675.282812] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.284314] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.285805] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91675.287132] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91675.288398] ofd_commitrw+0x5b4/0xd20 [ofd] [91675.289533] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.290874] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.292141] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.293458] ? flush_work+0x42/0x1d0 [91675.294442] ? internal_add_timer+0x42/0x70 [91675.295545] ? _cond_resched+0x15/0x30 [91675.296553] ? mutex_lock+0xe/0x30 [91675.297511] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.298926] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.300414] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.301937] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.303240] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.304572] kthread+0x134/0x150 [91675.305437] ? set_kthread_struct+0x50/0x50 [91675.306491] ret_from_fork+0x1f/0x40 [91675.307431] CPU: 9 PID: 527018 Comm: ll_ost_io02_006 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91675.309907] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91675.311912] Call Trace: [91675.312582] dump_stack+0x41/0x60 [91675.313593] warn_alloc.cold.127+0x7b/0x108 [91675.314688] __alloc_pages_slowpath+0xcb2/0xcd0 [91675.315799] ? __alloc_pages_nodemask+0x166/0x330 [91675.316984] __alloc_pages_nodemask+0x2e2/0x330 [91675.318169] kmalloc_order+0x28/0x90 [91675.319195] kmalloc_order_trace+0x1d/0xb0 [91675.320214] __kmalloc+0x203/0x250 [91675.321058] virtqueue_add+0x493/0xc70 [91675.322036] ? finish_wait+0x80/0x80 [91675.322947] virtqueue_add_sgs+0x80/0xa0 [91675.323866] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91675.325183] ? scsi_alloc_sgtables+0x84/0x1c0 [91675.326327] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91675.327579] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91675.328954] scsi_queue_rq+0x512/0xb10 [91675.329971] __blk_mq_try_issue_directly+0x163/0x200 [91675.331245] blk_mq_request_issue_directly+0x4e/0xb0 [91675.332492] blk_mq_try_issue_list_directly+0x62/0x100 [91675.333776] blk_mq_sched_insert_requests+0xa4/0xf0 [91675.335022] blk_mq_flush_plug_list+0x135/0x220 [91675.336194] blk_flush_plug_list+0xd7/0x100 [91675.337295] blk_finish_plug+0x25/0x36 [91675.338320] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91675.339794] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91675.341156] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.342725] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.344246] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91675.345591] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91675.346858] ofd_commitrw+0x5b4/0xd20 [ofd] [91675.347935] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.349241] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.350392] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.351726] ? flush_work+0x42/0x1d0 [91675.352723] ? internal_add_timer+0x42/0x70 [91675.353819] ? _cond_resched+0x15/0x30 [91675.354803] ? mutex_lock+0xe/0x30 [91675.355713] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.357109] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.358637] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.360157] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.361391] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.362709] kthread+0x134/0x150 [91675.363437] ? set_kthread_struct+0x50/0x50 [91675.364508] ret_from_fork+0x1f/0x40 [91675.365506] CPU: 13 PID: 527439 Comm: ll_ost_io03_054 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [91675.368484] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [91675.370586] Call Trace: [91675.371372] dump_stack+0x41/0x60 [91675.372281] warn_alloc.cold.127+0x7b/0x108 [91675.373369] __alloc_pages_slowpath+0xcb2/0xcd0 [91675.374534] ? __alloc_pages_nodemask+0x166/0x330 [91675.375756] __alloc_pages_nodemask+0x2e2/0x330 [91675.376895] kmalloc_order+0x28/0x90 [91675.377879] kmalloc_order_trace+0x1d/0xb0 [91675.378960] __kmalloc+0x203/0x250 [91675.379899] virtqueue_add+0x493/0xc70 [91675.380890] ? finish_wait+0x80/0x80 [91675.381850] virtqueue_add_sgs+0x80/0xa0 [91675.382915] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [91675.384275] ? scsi_alloc_sgtables+0x84/0x1c0 [91675.385431] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [91675.386730] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [91675.388148] scsi_queue_rq+0x512/0xb10 [91675.389195] __blk_mq_try_issue_directly+0x163/0x200 [91675.390367] blk_mq_request_issue_directly+0x4e/0xb0 [91675.391635] blk_mq_try_issue_list_directly+0x62/0x100 [91675.392940] blk_mq_sched_insert_requests+0xa4/0xf0 [91675.394230] blk_mq_flush_plug_list+0x135/0x220 [91675.395432] blk_flush_plug_list+0xd7/0x100 [91675.396577] blk_finish_plug+0x25/0x36 [91675.397672] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [91675.399135] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [91675.400423] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.401977] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [91675.403558] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [91675.404708] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [91675.405869] ofd_commitrw+0x5b4/0xd20 [ofd] [91675.406859] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.408199] obd_commitrw+0x1b6/0x370 [ptlrpc] [91675.409538] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [91675.410970] ? flush_work+0x42/0x1d0 [91675.412029] ? internal_add_timer+0x42/0x70 [91675.413150] ? _cond_resched+0x15/0x30 [91675.414186] ? mutex_lock+0xe/0x30 [91675.415162] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [91675.416612] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [91675.418192] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [91675.419779] ptlrpc_main+0xbec/0x1530 [ptlrpc] [91675.421099] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [91675.422413] kthread+0x134/0x150 [91675.423415] ? set_kthread_struct+0x50/0x50 [91675.424599] ret_from_fork+0x1f/0x40 [92408.267209] warn_alloc: 404 callbacks suppressed [92408.267214] ll_ost_io04_000: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [92408.272127] CPU: 17 PID: 188109 Comm: ll_ost_io04_000 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92408.275079] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92408.277295] Call Trace: [92408.278190] dump_stack+0x41/0x60 [92408.279214] warn_alloc.cold.127+0x7b/0x108 [92408.280371] __alloc_pages_slowpath+0xcb2/0xcd0 [92408.281623] ? __alloc_pages_nodemask+0x166/0x330 [92408.282922] __alloc_pages_nodemask+0x2e2/0x330 [92408.284156] kmalloc_order+0x28/0x90 [92408.285213] kmalloc_order_trace+0x1d/0xb0 [92408.286364] __kmalloc+0x203/0x250 [92408.287366] virtqueue_add+0x493/0xc70 [92408.288418] ? finish_wait+0x80/0x80 [92408.289443] virtqueue_add_sgs+0x80/0xa0 [92408.290577] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92408.292018] ? scsi_alloc_sgtables+0x84/0x1c0 [92408.293116] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92408.294368] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92408.295813] scsi_queue_rq+0x512/0xb10 [92408.296855] __blk_mq_try_issue_directly+0x163/0x200 [92408.298134] blk_mq_request_issue_directly+0x4e/0xb0 [92408.299436] blk_mq_try_issue_list_directly+0x62/0x100 [92408.300767] blk_mq_sched_insert_requests+0xa4/0xf0 [92408.302029] blk_mq_flush_plug_list+0x135/0x220 [92408.303208] blk_flush_plug_list+0xd7/0x100 [92408.304371] blk_finish_plug+0x25/0x36 [92408.305508] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92408.307110] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92408.308552] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92408.310189] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92408.311743] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92408.313128] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92408.314528] ofd_commitrw+0x5b4/0xd20 [ofd] [92408.315714] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92408.317106] obd_commitrw+0x1b6/0x370 [ptlrpc] [92408.318422] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92408.319833] ? kvm_set_wallclock+0x10/0x10 [92408.320932] ? sched_clock+0x5/0x10 [92408.321935] ? newidle_balance+0x1e2/0x3b0 [92408.323089] ? internal_add_timer+0x42/0x70 [92408.324250] ? _cond_resched+0x15/0x30 [92408.325275] ? mutex_lock+0xe/0x30 [92408.326249] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92408.327712] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92408.329288] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92408.330763] ? finish_wait+0x80/0x80 [92408.331681] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92408.332945] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92408.334275] kthread+0x134/0x150 [92408.335193] ? set_kthread_struct+0x50/0x50 [92408.336287] ret_from_fork+0x1f/0x40 [92408.337254] warn_alloc_show_mem: 9 callbacks suppressed [92408.337264] Mem-Info: [92408.339267] active_anon:27543 inactive_anon:360497 isolated_anon:0 active_file:30041474 inactive_file:3747904 isolated_file:5 unevictable:0 dirty:792 writeback:0 slab_reclaimable:787990 slab_unreclaimable:595676 mapped:20627 shmem:156 pagetables:5550 bounce:0 free:2452640 free_pcp:405 free_cma:0 [92408.347054] Node 0 active_anon:110172kB inactive_anon:1441988kB active_file:120182240kB inactive_file:14988004kB unevictable:0kB isolated(anon):0kB isolated(file):20kB mapped:82508kB dirty:3168kB writeback:0kB shmem:624kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35232kB pagetables:22200kB all_unreclaimable? no [92408.353043] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [92408.358146] lowmem_reserve[]: 0 913 150007 150007 150007 [92408.359492] Node 0 DMA32 free:596884kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:304kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [92408.364754] lowmem_reserve[]: 0 0 149093 149093 149093 [92408.366075] Node 0 Normal free:9199944kB min:49268kB low:201940kB high:354612kB active_anon:109792kB inactive_anon:1437628kB active_file:120186240kB inactive_file:14973312kB unevictable:0kB writepending:3168kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1780kB local_pcp:24kB free_cma:0kB [92408.371938] lowmem_reserve[]: 0 0 0 0 0 [92408.372955] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [92408.375599] Node 0 DMA32: 127*4kB (UMH) 115*8kB (UMH) 204*16kB (UMEH) 127*32kB (UMEH) 161*64kB (UMEH) 148*128kB (UME) 127*256kB (UMEH) 110*512kB (UMH) 85*1024kB (UME) 51*2048kB (UM) 68*4096kB (UM) = 596852kB [92408.379326] Node 0 Normal: 1157861*4kB (UME) 511295*8kB (UME) 29977*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 9201436kB [92408.381799] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [92408.383576] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [92408.385169] 33796818 total pagecache pages [92408.386094] 1871 pages in swap cache [92408.387041] Swap cache stats: add 1316331, delete 1314477, find 59261/71352 [92408.388550] Free swap = 6295672kB [92408.389447] Total swap = 11075580kB [92408.390282] 39321462 pages RAM [92408.391040] 0 pages HighMem/MovableOnly [92408.392035] 905848 pages reserved [92408.392825] 0 pages hwpoisoned [92542.668333] ll_ost_io00_003: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [92542.671716] CPU: 1 PID: 527120 Comm: ll_ost_io00_003 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92542.674817] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92542.676090] ll_ost_io04_014: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [92542.676104] ll_ost_io00_048: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [92542.676125] ll_ost_io00_006: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [92542.676127] ll_ost_io00_050: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [92542.676130] ,mems_allowed=0 [92542.676137] ,cpuset=/,mems_allowed=0 [92542.676963] Call Trace: [92542.677150] ll_ost_io04_009: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [92542.679580] ,cpuset= [92542.682118] dump_stack+0x41/0x60 [92542.684765] / [92542.686949] warn_alloc.cold.127+0x7b/0x108 [92542.686987] ll_ost_io03_061: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [92542.687996] ,mems_allowed=0 [92542.688021] ll_ost_io02_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [92542.689004] __alloc_pages_slowpath+0xcb2/0xcd0 [92542.689994] / [92542.689994] ,mems_allowed=0 [92542.692555] ? __alloc_pages_nodemask+0x166/0x330 [92542.693407] ,mems_allowed=0 [92542.694393] __alloc_pages_nodemask+0x2e2/0x330 [92542.696362] kmalloc_order+0x28/0x90 [92542.708973] kmalloc_order_trace+0x1d/0xb0 [92542.709995] __kmalloc+0x203/0x250 [92542.710891] virtqueue_add+0x493/0xc70 [92542.711862] ? finish_wait+0x80/0x80 [92542.712795] virtqueue_add_sgs+0x80/0xa0 [92542.712982] ll_ost_io02_082: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [92542.713776] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92542.717451] ? scsi_alloc_sgtables+0x84/0x1c0 [92542.718505] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92542.719667] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92542.720917] scsi_queue_rq+0x512/0xb10 [92542.721816] __blk_mq_try_issue_directly+0x163/0x200 [92542.722922] blk_mq_request_issue_directly+0x4e/0xb0 [92542.724015] blk_mq_try_issue_list_directly+0x62/0x100 [92542.725137] blk_mq_sched_insert_requests+0xa4/0xf0 [92542.726209] blk_mq_flush_plug_list+0x135/0x220 [92542.727235] blk_flush_plug_list+0xd7/0x100 [92542.728197] blk_finish_plug+0x25/0x36 [92542.729084] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92542.730407] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92542.731538] ll_ost_io02_063: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [92542.731595] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.735791] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.737133] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92542.738285] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92542.739356] ofd_commitrw+0x5b4/0xd20 [ofd] [92542.740317] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.741458] obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.742577] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92542.743711] ? flush_work+0x42/0x1d0 [92542.744555] ? internal_add_timer+0x42/0x70 [92542.745483] ? _cond_resched+0x15/0x30 [92542.746350] ? mutex_lock+0xe/0x30 [92542.747157] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92542.748334] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92542.749617] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92542.750933] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92542.752005] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92542.753188] kthread+0x134/0x150 [92542.753959] ? set_kthread_struct+0x50/0x50 [92542.754886] ret_from_fork+0x1f/0x40 [92542.755721] CPU: 2 PID: 527123 Comm: ll_ost_io00_006 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92542.755727] Mem-Info: [92542.758148] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92542.758867] active_anon:27544 inactive_anon:361136 isolated_anon:0 active_file:27684886 inactive_file:7954539 isolated_file:32 unevictable:0 dirty:1277 writeback:0 slab_reclaimable:799731 slab_unreclaimable:594853 mapped:23081 shmem:152 pagetables:5527 bounce:0 free:593165 free_pcp:503 free_cma:0 [92542.760666] Call Trace: [92542.767681] Node 0 active_anon:110176kB inactive_anon:1444544kB active_file:110739544kB inactive_file:31818156kB unevictable:0kB isolated(anon):0kB isolated(file):128kB mapped:92324kB dirty:5108kB writeback:0kB shmem:608kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35200kB pagetables:22108kB all_unreclaimable? no [92542.768373] dump_stack+0x41/0x60 [92542.774371] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [92542.775200] warn_alloc.cold.127+0x7b/0x108 [92542.779623] lowmem_reserve[]: 0 913 150007 150007 150007 [92542.780621] __alloc_pages_slowpath+0xcb2/0xcd0 [92542.781840] Node 0 DMA32 free:596580kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:1480kB inactive_file:244kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [92542.782884] ? __alloc_pages_nodemask+0x166/0x330 [92542.782888] lowmem_reserve[]: 0 [92542.787405] __alloc_pages_nodemask+0x2e2/0x330 [92542.788516] 0 [92542.789348] kmalloc_order+0x28/0x90 [92542.790440] 149093 [92542.791023] kmalloc_order_trace+0x1d/0xb0 [92542.791951] 149093 [92542.792590] __kmalloc+0x203/0x250 [92542.793589] 149093 [92542.794232] virtqueue_add+0x493/0xc70 [92542.795754] ? finish_wait+0x80/0x80 [92542.796681] Node 0 [92542.797223] virtqueue_add_sgs+0x80/0xa0 [92542.798134] Normal free:1764776kB min:49268kB low:201940kB high:354612kB active_anon:109796kB inactive_anon:1440204kB active_file:110738064kB inactive_file:31818220kB unevictable:0kB writepending:5108kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1856kB local_pcp:80kB free_cma:0kB [92542.798785] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92542.799764] lowmem_reserve[]: 0 [92542.804676] ? scsi_alloc_sgtables+0x84/0x1c0 [92542.805907] 0 [92542.806748] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92542.807825] 0 [92542.808423] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92542.809617] 0 [92542.810216] scsi_queue_rq+0x512/0xb10 [92542.811528] 0 [92542.812150] __blk_mq_try_issue_directly+0x163/0x200 [92542.813759] blk_mq_request_issue_directly+0x4e/0xb0 [92542.814962] Node 0 [92542.815551] blk_mq_try_issue_list_directly+0x62/0x100 [92542.816719] DMA: [92542.817373] blk_mq_sched_insert_requests+0xa4/0xf0 [92542.818566] 0*4kB [92542.819187] blk_mq_flush_plug_list+0x135/0x220 [92542.820329] 0*8kB [92542.820971] blk_flush_plug_list+0xd7/0x100 [92542.822053] 0*16kB [92542.822716] blk_finish_plug+0x25/0x36 [92542.823775] 0*32kB [92542.824440] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92542.825387] 0*64kB [92542.826037] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92542.827322] 0*128kB [92542.827965] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.829135] 0*256kB [92542.829801] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.831175] 0*512kB [92542.831845] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92542.833206] 1*1024kB [92542.833871] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92542.835032] (U) [92542.835689] ofd_commitrw+0x5b4/0xd20 [ofd] [92542.836801] 1*2048kB [92542.837409] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.838362] (M) [92542.839022] obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.840052] 2*4096kB [92542.840630] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92542.841640] (M) [92542.842303] ? newidle_balance+0x2b6/0x3b0 [92542.843381] = 11264kB [92542.843968] ? internal_add_timer+0x42/0x70 [92542.844971] Node 0 [92542.845629] ? _cond_resched+0x15/0x30 [92542.846636] DMA32: [92542.847262] ? mutex_lock+0xe/0x30 [92542.848187] 143*4kB [92542.848807] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92542.849636] (UMH) [92542.850266] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92542.851363] 111*8kB [92542.851984] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92542.853224] (UMH) [92542.853868] ? finish_wait+0x80/0x80 [92542.855133] 171*16kB [92542.855743] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92542.856584] (UMEH) [92542.857230] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92542.858218] 94*32kB [92542.858838] kthread+0x134/0x150 [92542.859963] (UMEH) [92542.860582] ? set_kthread_struct+0x50/0x50 [92542.861392] 147*64kB [92542.862005] ret_from_fork+0x1f/0x40 [92542.862981] (UME) [92542.863617] CPU: 8 PID: 527702 Comm: ll_ost_io02_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92542.864406] 151*128kB [92542.865080] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92542.867557] (UME) [92542.868238] Call Trace: [92542.870091] 128*256kB [92542.870717] dump_stack+0x41/0x60 [92542.871413] (UME) [92542.872083] warn_alloc.cold.127+0x7b/0x108 [92542.872930] 111*512kB [92542.873544] __alloc_pages_slowpath+0xcb2/0xcd0 [92542.874531] (UMH) [92542.875207] ? __alloc_pages_nodemask+0x166/0x330 [92542.876249] 86*1024kB [92542.876851] __alloc_pages_nodemask+0x2e2/0x330 [92542.877928] (UMEH) [92542.878603] kmalloc_order+0x28/0x90 [92542.879642] 51*2048kB [92542.880259] kmalloc_order_trace+0x1d/0xb0 [92542.881141] (UM) [92542.881791] __kmalloc+0x203/0x250 [92542.882757] 68*4096kB [92542.883346] virtqueue_add+0x493/0xc70 [92542.884201] (UM) [92542.884857] virtqueue_add_sgs+0x80/0xa0 [92542.885768] = 596580kB [92542.886355] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92542.887300] Node 0 [92542.887962] ? scsi_alloc_sgtables+0x84/0x1c0 [92542.889145] Normal: [92542.889770] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92542.890777] 358944*4kB [92542.891439] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92542.892577] (UME) [92542.893262] scsi_queue_rq+0x512/0xb10 [92542.894505] 41492*8kB [92542.895122] __blk_mq_try_issue_directly+0x163/0x200 [92542.896044] (UME) [92542.896716] blk_mq_request_issue_directly+0x4e/0xb0 [92542.897832] 536*16kB [92542.898441] blk_mq_try_issue_list_directly+0x62/0x100 [92542.899557] (UEH) [92542.900213] blk_mq_sched_insert_requests+0xa4/0xf0 [92542.901361] 10*32kB [92542.901977] blk_mq_flush_plug_list+0x135/0x220 [92542.903107] (UH) [92542.903749] blk_flush_plug_list+0xd7/0x100 [92542.904805] 0*64kB [92542.905417] blk_finish_plug+0x25/0x36 [92542.906437] 0*128kB [92542.907091] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92542.908016] 0*256kB [92542.908666] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92542.909921] 0*512kB [92542.910553] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.911712] 0*1024kB [92542.912353] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.913697] 0*2048kB [92542.914341] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92542.915645] 0*4096kB [92542.916287] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92542.917426] = 1776608kB [92542.918070] ofd_commitrw+0x5b4/0xd20 [ofd] [92542.919146] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [92542.919816] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.920742] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [92542.922407] obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.923420] 35636322 total pagecache pages [92542.925045] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92542.926035] 1543 pages in swap cache [92542.926992] ? flush_work+0x42/0x1d0 [92542.928063] Swap cache stats: add 1318392, delete 1316866, find 59964/72184 [92542.928910] ? internal_add_timer+0x42/0x70 [92542.929791] Free swap = 6293368kB [92542.931185] ? _cond_resched+0x15/0x30 [92542.932174] Total swap = 11075580kB [92542.933001] ? mutex_lock+0xe/0x30 [92542.933928] 39321462 pages RAM [92542.934768] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92542.935591] 0 pages HighMem/MovableOnly [92542.936370] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92542.937481] 905848 pages reserved [92542.938397] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92542.939602] 0 pages hwpoisoned [92542.940419] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92542.943329] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92542.944483] kthread+0x134/0x150 [92542.945210] ? set_kthread_struct+0x50/0x50 [92542.946083] ret_from_fork+0x1f/0x40 [92542.946862] CPU: 3 PID: 527425 Comm: ll_ost_io00_048 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92542.948869] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92542.950391] Call Trace: [92542.950917] dump_stack+0x41/0x60 [92542.951565] warn_alloc.cold.127+0x7b/0x108 [92542.952329] __alloc_pages_slowpath+0xcb2/0xcd0 [92542.953154] ? __alloc_pages_nodemask+0x166/0x330 [92542.953994] __alloc_pages_nodemask+0x2e2/0x330 [92542.954802] kmalloc_order+0x28/0x90 [92542.955475] kmalloc_order_trace+0x1d/0xb0 [92542.956226] __kmalloc+0x203/0x250 [92542.956914] virtqueue_add+0x493/0xc70 [92542.957642] ? finish_wait+0x80/0x80 [92542.958345] virtqueue_add_sgs+0x80/0xa0 [92542.959069] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92542.960008] ? scsi_alloc_sgtables+0x84/0x1c0 [92542.960810] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92542.961686] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92542.962648] scsi_queue_rq+0x512/0xb10 [92542.963341] __blk_mq_try_issue_directly+0x163/0x200 [92542.964220] blk_mq_request_issue_directly+0x4e/0xb0 [92542.965083] blk_mq_try_issue_list_directly+0x62/0x100 [92542.965969] blk_mq_sched_insert_requests+0xa4/0xf0 [92542.966816] blk_mq_flush_plug_list+0x135/0x220 [92542.967616] blk_flush_plug_list+0xd7/0x100 [92542.968365] blk_finish_plug+0x25/0x36 [92542.969068] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92542.970107] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.971174] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92542.972220] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92542.973147] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92542.974005] ofd_commitrw+0x5b4/0xd20 [ofd] [92542.974769] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.975677] obd_commitrw+0x1b6/0x370 [ptlrpc] [92542.976562] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92542.977467] ? newidle_balance+0x2b6/0x3b0 [92542.978220] ? internal_add_timer+0x42/0x70 [92542.978972] ? _cond_resched+0x15/0x30 [92542.979669] ? mutex_lock+0xe/0x30 [92542.980316] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92542.981294] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92542.982343] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92542.983396] ? finish_wait+0x80/0x80 [92542.984061] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92542.984911] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92542.985872] kthread+0x134/0x150 [92542.986506] ? set_kthread_struct+0x50/0x50 [92542.987265] ret_from_fork+0x1f/0x40 [92542.987932] CPU: 14 PID: 527446 Comm: ll_ost_io03_061 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92542.990016] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92542.991519] Call Trace: [92542.992039] dump_stack+0x41/0x60 [92542.992685] warn_alloc.cold.127+0x7b/0x108 [92542.993445] __alloc_pages_slowpath+0xcb2/0xcd0 [92542.994254] __alloc_pages_nodemask+0x2e2/0x330 [92542.995067] kmalloc_order+0x28/0x90 [92542.995745] kmalloc_order_trace+0x1d/0xb0 [92542.996492] __kmalloc+0x203/0x250 [92542.997147] virtqueue_add+0x493/0xc70 [92542.997838] ? finish_wait+0x80/0x80 [92542.998510] virtqueue_add_sgs+0x80/0xa0 [92542.999227] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92543.000153] ? scsi_alloc_sgtables+0x84/0x1c0 [92543.000930] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92543.001813] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92543.002774] scsi_queue_rq+0x512/0xb10 [92543.003458] __blk_mq_try_issue_directly+0x163/0x200 [92543.004356] blk_mq_request_issue_directly+0x4e/0xb0 [92543.005236] blk_mq_try_issue_list_directly+0x62/0x100 [92543.006129] blk_mq_sched_insert_requests+0xa4/0xf0 [92543.006973] blk_mq_flush_plug_list+0x135/0x220 [92543.007774] blk_flush_plug_list+0xd7/0x100 [92543.008518] blk_finish_plug+0x25/0x36 [92543.009197] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92543.010221] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92543.011178] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.012257] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.013315] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92543.014227] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92543.015071] ofd_commitrw+0x5b4/0xd20 [ofd] [92543.015832] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.016768] obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.017677] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92543.018590] ? newidle_balance+0x2b6/0x3b0 [92543.019334] ? internal_add_timer+0x42/0x70 [92543.020092] ? _cond_resched+0x15/0x30 [92543.020779] ? mutex_lock+0xe/0x30 [92543.021426] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92543.022357] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92543.023403] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92543.024458] ? finish_wait+0x80/0x80 [92543.025894] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92543.026788] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92543.027733] kthread+0x134/0x150 [92543.028348] ? set_kthread_struct+0x50/0x50 [92543.029121] ret_from_fork+0x1f/0x40 [92543.029801] CPU: 16 PID: 527215 Comm: ll_ost_io04_014 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92543.032538] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92543.034617] Call Trace: [92543.035479] dump_stack+0x41/0x60 [92543.036425] warn_alloc.cold.127+0x7b/0x108 [92543.037518] __alloc_pages_slowpath+0xcb2/0xcd0 [92543.038660] ? __alloc_pages_nodemask+0x166/0x330 [92543.039830] __alloc_pages_nodemask+0x2e2/0x330 [92543.040976] kmalloc_order+0x28/0x90 [92543.041971] kmalloc_order_trace+0x1d/0xb0 [92543.043103] __kmalloc+0x203/0x250 [92543.044046] virtqueue_add+0x493/0xc70 [92543.045051] ? finish_wait+0x80/0x80 [92543.046006] virtqueue_add_sgs+0x80/0xa0 [92543.046993] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92543.048272] ? scsi_alloc_sgtables+0x84/0x1c0 [92543.049365] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92543.050618] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92543.051997] scsi_queue_rq+0x512/0xb10 [92543.052994] __blk_mq_try_issue_directly+0x163/0x200 [92543.054203] blk_mq_request_issue_directly+0x4e/0xb0 [92543.055445] blk_mq_try_issue_list_directly+0x62/0x100 [92543.056755] blk_mq_sched_insert_requests+0xa4/0xf0 [92543.057888] blk_mq_flush_plug_list+0x135/0x220 [92543.058807] blk_flush_plug_list+0xd7/0x100 [92543.059703] blk_finish_plug+0x25/0x36 [92543.060615] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92543.061760] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92543.062856] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.064040] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.065210] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92543.066366] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92543.067233] ofd_commitrw+0x5b4/0xd20 [ofd] [92543.067993] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.068884] obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.069758] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92543.070646] ? flush_work+0x42/0x1d0 [92543.071313] ? internal_add_timer+0x42/0x70 [92543.072065] ? _cond_resched+0x15/0x30 [92543.072761] ? mutex_lock+0xe/0x30 [92543.073401] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92543.074358] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92543.075410] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92543.076496] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92543.077413] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92543.078399] kthread+0x134/0x150 [92543.079075] ? set_kthread_struct+0x50/0x50 [92543.079875] ret_from_fork+0x1f/0x40 [92543.080617] CPU: 10 PID: 527338 Comm: ll_ost_io02_063 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92543.082716] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92543.084317] Call Trace: [92543.084872] dump_stack+0x41/0x60 [92543.085549] warn_alloc.cold.127+0x7b/0x108 [92543.086375] __alloc_pages_slowpath+0xcb2/0xcd0 [92543.087214] __alloc_pages_nodemask+0x2e2/0x330 [92543.088050] kmalloc_order+0x28/0x90 [92543.088738] kmalloc_order_trace+0x1d/0xb0 [92543.089493] __kmalloc+0x203/0x250 [92543.090155] virtqueue_add+0x493/0xc70 [92543.090856] ? finish_wait+0x80/0x80 [92543.091546] virtqueue_add_sgs+0x80/0xa0 [92543.092277] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92543.093238] ? scsi_alloc_sgtables+0x84/0x1c0 [92543.094051] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92543.094938] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92543.095918] scsi_queue_rq+0x512/0xb10 [92543.096616] __blk_mq_try_issue_directly+0x163/0x200 [92543.097492] blk_mq_request_issue_directly+0x4e/0xb0 [92543.098361] blk_mq_try_issue_list_directly+0x62/0x100 [92543.099269] blk_mq_sched_insert_requests+0xa4/0xf0 [92543.100136] blk_mq_flush_plug_list+0x135/0x220 [92543.100962] blk_flush_plug_list+0xd7/0x100 [92543.101736] blk_finish_plug+0x25/0x36 [92543.102434] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92543.103454] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92543.104390] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.105548] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.106673] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92543.107634] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92543.108518] ofd_commitrw+0x5b4/0xd20 [ofd] [92543.109306] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.110354] obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.111353] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92543.112416] ? flush_work+0x42/0x1d0 [92543.113252] ? internal_add_timer+0x42/0x70 [92543.114070] ? _cond_resched+0x15/0x30 [92543.114837] ? mutex_lock+0xe/0x30 [92543.115707] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92543.116876] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92543.117965] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92543.119071] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92543.119954] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92543.120938] kthread+0x134/0x150 [92543.121594] ? set_kthread_struct+0x50/0x50 [92543.122386] ret_from_fork+0x1f/0x40 [92543.123071] CPU: 9 PID: 527875 Comm: ll_ost_io02_082 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92543.125733] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92543.127669] Call Trace: [92543.128408] dump_stack+0x41/0x60 [92543.129095] warn_alloc.cold.127+0x7b/0x108 [92543.129883] __alloc_pages_slowpath+0xcb2/0xcd0 [92543.130704] ? __alloc_pages_nodemask+0x2e2/0x330 [92543.131532] __alloc_pages_nodemask+0x2e2/0x330 [92543.132332] kmalloc_order+0x28/0x90 [92543.133016] kmalloc_order_trace+0x1d/0xb0 [92543.133767] __kmalloc+0x203/0x250 [92543.134416] ? __wake_up_common+0x7a/0x190 [92543.135161] virtqueue_add+0x493/0xc70 [92543.135858] ? finish_wait+0x80/0x80 [92543.136533] virtqueue_add_sgs+0x80/0xa0 [92543.137245] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92543.138170] ? scsi_alloc_sgtables+0x84/0x1c0 [92543.138991] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92543.139897] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92543.140870] scsi_queue_rq+0x512/0xb10 [92543.141564] __blk_mq_try_issue_directly+0x163/0x200 [92543.142432] blk_mq_request_issue_directly+0x4e/0xb0 [92543.143292] blk_mq_try_issue_list_directly+0x62/0x100 [92543.144192] blk_mq_sched_insert_requests+0xa4/0xf0 [92543.145039] blk_mq_flush_plug_list+0x135/0x220 [92543.145847] blk_flush_plug_list+0xd7/0x100 [92543.146607] blk_finish_plug+0x25/0x36 [92543.147325] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92543.148367] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92543.149298] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.150418] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.151571] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92543.152527] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92543.153396] ofd_commitrw+0x5b4/0xd20 [ofd] [92543.154195] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.155120] obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.156030] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92543.156976] ? flush_work+0x42/0x1d0 [92543.157688] ? internal_add_timer+0x42/0x70 [92543.158479] ? _cond_resched+0x15/0x30 [92543.159209] ? mutex_lock+0xe/0x30 [92543.159890] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92543.160922] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92543.162336] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92543.163505] ? finish_wait+0x80/0x80 [92543.164194] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92543.165096] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92543.166091] kthread+0x134/0x150 [92543.166760] ? set_kthread_struct+0x50/0x50 [92543.167565] ret_from_fork+0x1f/0x40 [92543.168341] CPU: 19 PID: 527178 Comm: ll_ost_io04_009 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92543.171127] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92543.173174] Call Trace: [92543.173942] dump_stack+0x41/0x60 [92543.174921] warn_alloc.cold.127+0x7b/0x108 [92543.176048] __alloc_pages_slowpath+0xcb2/0xcd0 [92543.177208] ? __alloc_pages_nodemask+0x166/0x330 [92543.178375] __alloc_pages_nodemask+0x2e2/0x330 [92543.179506] kmalloc_order+0x28/0x90 [92543.180458] kmalloc_order_trace+0x1d/0xb0 [92543.181516] __kmalloc+0x203/0x250 [92543.182463] virtqueue_add+0x493/0xc70 [92543.183451] ? finish_wait+0x80/0x80 [92543.184425] virtqueue_add_sgs+0x80/0xa0 [92543.185435] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92543.186750] ? scsi_alloc_sgtables+0x84/0x1c0 [92543.187861] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92543.189156] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92543.190551] scsi_queue_rq+0x512/0xb10 [92543.191533] __blk_mq_try_issue_directly+0x163/0x200 [92543.192753] blk_mq_request_issue_directly+0x4e/0xb0 [92543.193975] blk_mq_try_issue_list_directly+0x62/0x100 [92543.195227] blk_mq_sched_insert_requests+0xa4/0xf0 [92543.196437] blk_mq_flush_plug_list+0x135/0x220 [92543.197588] blk_flush_plug_list+0xd7/0x100 [92543.198660] blk_finish_plug+0x25/0x36 [92543.199650] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92543.201109] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92543.202454] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.204001] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.205497] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92543.206826] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92543.208077] ofd_commitrw+0x5b4/0xd20 [ofd] [92543.209204] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.210548] obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.211871] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92543.213239] ? flush_work+0x42/0x1d0 [92543.214246] ? internal_add_timer+0x42/0x70 [92543.215376] ? _cond_resched+0x15/0x30 [92543.216412] ? mutex_lock+0xe/0x30 [92543.217371] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92543.218798] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92543.220344] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92543.221943] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92543.223264] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92543.224681] kthread+0x134/0x150 [92543.225606] ? set_kthread_struct+0x50/0x50 [92543.226742] ret_from_fork+0x1f/0x40 [92543.227736] CPU: 0 PID: 527427 Comm: ll_ost_io00_050 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [92543.230520] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [92543.232237] Call Trace: [92543.232879] dump_stack+0x41/0x60 [92543.233602] warn_alloc.cold.127+0x7b/0x108 [92543.234362] __alloc_pages_slowpath+0xcb2/0xcd0 [92543.235207] ? __alloc_pages_nodemask+0x166/0x330 [92543.236115] __alloc_pages_nodemask+0x2e2/0x330 [92543.237194] kmalloc_order+0x28/0x90 [92543.237965] kmalloc_order_trace+0x1d/0xb0 [92543.238977] __kmalloc+0x203/0x250 [92543.239871] ? __wake_up_common+0x7a/0x190 [92543.241023] virtqueue_add+0x493/0xc70 [92543.242081] ? finish_wait+0x80/0x80 [92543.243052] virtqueue_add_sgs+0x80/0xa0 [92543.244095] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [92543.245461] ? scsi_alloc_sgtables+0x84/0x1c0 [92543.246599] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [92543.247540] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [92543.248614] scsi_queue_rq+0x512/0xb10 [92543.249365] __blk_mq_try_issue_directly+0x163/0x200 [92543.250236] blk_mq_request_issue_directly+0x4e/0xb0 [92543.251123] blk_mq_try_issue_list_directly+0x62/0x100 [92543.252091] blk_mq_sched_insert_requests+0xa4/0xf0 [92543.253186] blk_mq_flush_plug_list+0x135/0x220 [92543.254312] blk_flush_plug_list+0xd7/0x100 [92543.255415] blk_finish_plug+0x25/0x36 [92543.256192] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [92543.257499] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [92543.258786] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.260317] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [92543.262180] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [92543.263516] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [92543.264785] ofd_commitrw+0x5b4/0xd20 [ofd] [92543.265924] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.267287] obd_commitrw+0x1b6/0x370 [ptlrpc] [92543.268614] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [92543.269827] ? flush_work+0x42/0x1d0 [92543.270732] ? internal_add_timer+0x42/0x70 [92543.271845] ? _cond_resched+0x15/0x30 [92543.272844] ? mutex_lock+0xe/0x30 [92543.273779] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [92543.275447] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [92543.277014] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [92543.278554] ptlrpc_main+0xbec/0x1530 [ptlrpc] [92543.279832] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [92543.281220] kthread+0x134/0x150 [92543.282128] ? set_kthread_struct+0x50/0x50 [92543.283104] ret_from_fork+0x1f/0x40 [93069.869872] ll_ost_io01_080: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [93069.869871] ll_ost_io01_053: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [93069.869880] ll_ost_io01_052: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [93069.869883] ,cpuset= [93069.869884] ,cpuset= [93069.869888] / [93069.869890] ll_ost_io02_041: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [93069.869913] ll_ost_io01_055: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [93069.872473] ,cpuset= [93069.873442] ll_ost_io00_060: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [93069.874604] / [93069.874603] ,mems_allowed=0 [93069.874615] CPU: 5 PID: 527157 Comm: ll_ost_io01_053 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93069.874621] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93069.874623] Call Trace: [93069.874649] dump_stack+0x41/0x60 [93069.874667] warn_alloc.cold.127+0x7b/0x108 [93069.874673] __alloc_pages_slowpath+0xcb2/0xcd0 [93069.874677] ? __alloc_pages_nodemask+0x166/0x330 [93069.874680] __alloc_pages_nodemask+0x2e2/0x330 [93069.874682] kmalloc_order+0x28/0x90 [93069.874688] kmalloc_order_trace+0x1d/0xb0 [93069.874691] __kmalloc+0x203/0x250 [93069.874696] virtqueue_add+0x493/0xc70 [93069.874701] ? finish_wait+0x80/0x80 [93069.874705] virtqueue_add_sgs+0x80/0xa0 [93069.874719] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93069.874735] ? scsi_alloc_sgtables+0x84/0x1c0 [93069.874740] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93069.874743] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93069.874746] scsi_queue_rq+0x512/0xb10 [93069.874749] __blk_mq_try_issue_directly+0x163/0x200 [93069.874755] blk_mq_request_issue_directly+0x4e/0xb0 [93069.874758] blk_mq_try_issue_list_directly+0x62/0x100 [93069.874762] blk_mq_sched_insert_requests+0xa4/0xf0 [93069.874765] blk_mq_flush_plug_list+0x135/0x220 [93069.874769] blk_flush_plug_list+0xd7/0x100 [93069.874774] blk_finish_plug+0x25/0x36 [93069.874784] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93069.874878] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93069.874973] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.874997] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.875023] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93069.875049] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93069.875105] ofd_commitrw+0x5b4/0xd20 [ofd] [93069.875121] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.875235] obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.875340] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93069.875438] ? newidle_balance+0x2b6/0x3b0 [93069.875442] ? internal_add_timer+0x42/0x70 [93069.875445] ? _cond_resched+0x15/0x30 [93069.875450] ? mutex_lock+0xe/0x30 [93069.875455] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93069.875566] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93069.875670] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93069.875779] ? finish_wait+0x80/0x80 [93069.875782] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93069.875883] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93069.875990] kthread+0x134/0x150 [93069.875995] ? set_kthread_struct+0x50/0x50 [93069.875998] ret_from_fork+0x1f/0x40 [93069.876003] warn_alloc_show_mem: 9 callbacks suppressed [93069.876008] Mem-Info: [93069.876011] active_anon:27649 inactive_anon:339222 isolated_anon:0 active_file:29505883 inactive_file:5510559 isolated_file:0 unevictable:0 dirty:733 writeback:0 slab_reclaimable:835117 slab_unreclaimable:617547 mapped:21878 shmem:148 pagetables:5550 bounce:0 free:1181615 free_pcp:505 free_cma:0 [93069.876016] Node 0 active_anon:110596kB inactive_anon:1356888kB active_file:118023532kB inactive_file:22042236kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:87512kB dirty:2932kB writeback:0kB shmem:592kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35264kB pagetables:22200kB all_unreclaimable? no [93069.876024] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [93069.876030] lowmem_reserve[]: 0 913 150007 150007 150007 [93069.876034] Node 0 DMA32 free:596512kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4360kB active_file:24kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [93069.876042] lowmem_reserve[]: 0 0 149093 149093 149093 [93069.876047] Node 0 Normal free:4118712kB min:49268kB low:201940kB high:354612kB active_anon:110216kB inactive_anon:1352528kB active_file:118023340kB inactive_file:22042236kB unevictable:0kB writepending:2932kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2024kB local_pcp:108kB free_cma:0kB [93069.876052] lowmem_reserve[]: 0 0 0 0 0 [93069.876056] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [93069.876068] Node 0 DMA32: 158*4kB (UM) 123*8kB (UM) 167*16kB (UME) 86*32kB (UMEH) 141*64kB (UM) 138*128kB (UMEH) 118*256kB (UME) 104*512kB (UMEH) 80*1024kB (UMH) 52*2048kB (UM) 71*4096kB (UM) = 596416kB [93069.876085] Node 0 Normal: 733225*4kB (UME) 127383*8kB (UME) 10434*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 4118908kB [93069.876098] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [93069.876100] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [93069.876102] 35016350 total pagecache pages [93069.876103] 1641 pages in swap cache [93069.876104] Swap cache stats: add 1355133, delete 1353509, find 62554/75299 [93069.876105] Free swap = 6167096kB [93069.876106] Total swap = 11075580kB [93069.876107] 39321462 pages RAM [93069.876107] 0 pages HighMem/MovableOnly [93069.876108] 905848 pages reserved [93069.876109] 0 pages hwpoisoned [93069.883125] ll_ost_io01_008: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [93069.883237] ,mems_allowed=0 [93069.883239] /,mems_allowed=0 [93069.883242] /,mems_allowed=0 [93069.883244] CPU: 9 PID: 527246 Comm: ll_ost_io02_041 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93069.883245] / [93069.883248] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93069.883247] ,mems_allowed=0 [93069.883248] / [93069.883249] Call Trace: [93069.883250] ,mems_allowed=0 [93069.883253] dump_stack+0x41/0x60 [93069.883260] warn_alloc.cold.127+0x7b/0x108 [93069.883265] __alloc_pages_slowpath+0xcb2/0xcd0 [93069.883268] ? __alloc_pages_nodemask+0x166/0x330 [93069.883271] __alloc_pages_nodemask+0x2e2/0x330 [93069.883273] kmalloc_order+0x28/0x90 [93069.883278] kmalloc_order_trace+0x1d/0xb0 [93069.883281] __kmalloc+0x203/0x250 [93069.883285] virtqueue_add+0x493/0xc70 [93069.883290] virtqueue_add_sgs+0x80/0xa0 [93069.883294] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93069.883298] ? scsi_alloc_sgtables+0x84/0x1c0 [93069.883303] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93069.883306] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93069.883309] scsi_queue_rq+0x512/0xb10 [93069.883311] __blk_mq_try_issue_directly+0x163/0x200 [93069.883316] blk_mq_request_issue_directly+0x4e/0xb0 [93069.883320] blk_mq_try_issue_list_directly+0x62/0x100 [93069.883323] blk_mq_sched_insert_requests+0xa4/0xf0 [93069.883326] blk_mq_flush_plug_list+0x135/0x220 [93069.883330] blk_flush_plug_list+0xd7/0x100 [93069.883334] blk_finish_plug+0x25/0x36 [93069.883338] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93069.883365] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93069.883387] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.883405] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.883424] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93069.883442] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93069.883457] ofd_commitrw+0x5b4/0xd20 [ofd] [93069.883468] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.883553] obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.883621] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93069.883689] ? flush_work+0x42/0x1d0 [93069.883694] ? internal_add_timer+0x42/0x70 [93069.883697] ? _cond_resched+0x15/0x30 [93069.883699] ? mutex_lock+0xe/0x30 [93069.883702] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93069.883769] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93069.883835] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93069.883898] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93069.883962] ,cpuset= [93069.883961] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93069.884026] kthread+0x134/0x150 [93069.884029] ? set_kthread_struct+0x50/0x50 [93069.884032] ret_from_fork+0x1f/0x40 [93069.884037] CPU: 7 PID: 528149 Comm: ll_ost_io01_080 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93069.884041] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93069.884043] Call Trace: [93069.884046] dump_stack+0x41/0x60 [93069.884053] warn_alloc.cold.127+0x7b/0x108 [93069.884059] __alloc_pages_slowpath+0xcb2/0xcd0 [93069.884062] ? __alloc_pages_nodemask+0x166/0x330 [93069.884066] __alloc_pages_nodemask+0x2e2/0x330 [93069.884069] kmalloc_order+0x28/0x90 [93069.884074] kmalloc_order_trace+0x1d/0xb0 [93069.884078] __kmalloc+0x203/0x250 [93069.884083] virtqueue_add+0x493/0xc70 [93069.884089] ? finish_wait+0x80/0x80 [93069.884092] virtqueue_add_sgs+0x80/0xa0 [93069.884097] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93069.884103] ? scsi_alloc_sgtables+0x84/0x1c0 [93069.884109] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93069.884112] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93069.884116] scsi_queue_rq+0x512/0xb10 [93069.884119] __blk_mq_try_issue_directly+0x163/0x200 [93069.884125] blk_mq_request_issue_directly+0x4e/0xb0 [93069.884130] blk_mq_try_issue_list_directly+0x62/0x100 [93069.884134] blk_mq_sched_insert_requests+0xa4/0xf0 [93069.884137] blk_mq_flush_plug_list+0x135/0x220 [93069.884142] blk_flush_plug_list+0xd7/0x100 [93069.884148] blk_finish_plug+0x25/0x36 [93069.884153] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93069.884191] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93069.884222] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.884251] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.884281] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93069.884310] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93069.884330] ofd_commitrw+0x5b4/0xd20 [ofd] [93069.884350] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.884474] obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.884560] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93069.884677] ? newidle_balance+0x2b6/0x3b0 [93069.884681] ? internal_add_timer+0x42/0x70 [93069.884685] ? _cond_resched+0x15/0x30 [93069.884687] ? mutex_lock+0xe/0x30 [93069.884690] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93069.884804] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93069.884925] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93069.885041] ? finish_wait+0x80/0x80 [93069.885044] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93069.885165] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93069.885285] kthread+0x134/0x150 [93069.885289] ? set_kthread_struct+0x50/0x50 [93069.885292] ret_from_fork+0x1f/0x40 [93069.885298] CPU: 4 PID: 527159 Comm: ll_ost_io01_055 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93069.885302] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93069.885303] Call Trace: [93069.885305] dump_stack+0x41/0x60 [93069.885310] warn_alloc.cold.127+0x7b/0x108 [93069.885315] __alloc_pages_slowpath+0xcb2/0xcd0 [93069.885318] ? __alloc_pages_nodemask+0x166/0x330 [93069.885321] __alloc_pages_nodemask+0x2e2/0x330 [93069.885323] kmalloc_order+0x28/0x90 [93069.885328] kmalloc_order_trace+0x1d/0xb0 [93069.885331] __kmalloc+0x203/0x250 [93069.885334] virtqueue_add+0x493/0xc70 [93069.885339] ? finish_wait+0x80/0x80 [93069.885342] virtqueue_add_sgs+0x80/0xa0 [93069.885346] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93069.885350] ? scsi_alloc_sgtables+0x84/0x1c0 [93069.885355] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93069.885358] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93069.885361] scsi_queue_rq+0x512/0xb10 [93069.885364] __blk_mq_try_issue_directly+0x163/0x200 [93069.885369] blk_mq_request_issue_directly+0x4e/0xb0 [93069.885373] blk_mq_try_issue_list_directly+0x62/0x100 [93069.885376] blk_mq_sched_insert_requests+0xa4/0xf0 [93069.885379] blk_mq_flush_plug_list+0x135/0x220 [93069.885383] blk_flush_plug_list+0xd7/0x100 [93069.885387] blk_finish_plug+0x25/0x36 [93069.885391] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93069.885424] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93069.885451] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.885475] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.885499] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93069.885523] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93069.885541] ofd_commitrw+0x5b4/0xd20 [ofd] [93069.885556] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.885668] obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.885771] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93069.885877] ? internal_add_timer+0x42/0x70 [93069.885880] ? _cond_resched+0x15/0x30 [93069.885883] ? mutex_lock+0xe/0x30 [93069.885885] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93069.885991] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93069.886096] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93069.886208] ? finish_wait+0x80/0x80 [93069.886211] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93069.886330] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93069.886431] kthread+0x134/0x150 [93069.886434] ? set_kthread_struct+0x50/0x50 [93069.886437] ret_from_fork+0x1f/0x40 [93069.886442] CPU: 1 PID: 527753 Comm: ll_ost_io00_060 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93069.886446] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93069.886448] Call Trace: [93069.886451] dump_stack+0x41/0x60 [93069.886456] warn_alloc.cold.127+0x7b/0x108 [93069.886460] __alloc_pages_slowpath+0xcb2/0xcd0 [93069.886463] ? __alloc_pages_nodemask+0x166/0x330 [93069.886467] __alloc_pages_nodemask+0x2e2/0x330 [93069.886469] kmalloc_order+0x28/0x90 [93069.886473] kmalloc_order_trace+0x1d/0xb0 [93069.886476] __kmalloc+0x203/0x250 [93069.886480] virtqueue_add+0x493/0xc70 [93069.886485] ? finish_wait+0x80/0x80 [93069.886487] virtqueue_add_sgs+0x80/0xa0 [93069.886491] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93069.886496] ? scsi_alloc_sgtables+0x84/0x1c0 [93069.886501] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93069.886504] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93069.886507] scsi_queue_rq+0x512/0xb10 [93069.886509] __blk_mq_try_issue_directly+0x163/0x200 [93069.886514] blk_mq_request_issue_directly+0x4e/0xb0 [93069.886518] blk_mq_try_issue_list_directly+0x62/0x100 [93069.886521] blk_mq_sched_insert_requests+0xa4/0xf0 [93069.886524] blk_mq_flush_plug_list+0x135/0x220 [93069.886528] blk_flush_plug_list+0xd7/0x100 [93069.886532] blk_finish_plug+0x25/0x36 [93069.886537] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93069.886572] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93069.886603] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.886621] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93069.886640] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93069.886658] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93069.886672] ofd_commitrw+0x5b4/0xd20 [ofd] [93069.886684] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.886766] obd_commitrw+0x1b6/0x370 [ptlrpc] [93069.886856] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93069.886930] ? flush_work+0x42/0x1d0 [93069.886935] ? internal_add_timer+0x42/0x70 [93069.886937] ? _cond_resched+0x15/0x30 [93069.886939] ? mutex_lock+0xe/0x30 [93069.886941] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93069.887024] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93069.887113] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93069.887213] ? osd_it_ea_store+0x20/0x20 [osd_ldiskfs] [93069.887231] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93069.887302] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93069.887377] kthread+0x134/0x150 [93069.887380] ? set_kthread_struct+0x50/0x50 [93069.887382] ret_from_fork+0x1f/0x40 [93069.887609] / [93069.888363] CPU: 6 PID: 527156 Comm: ll_ost_io01_052 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93069.891323] ,mems_allowed=0 [93069.892987] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93069.894637] Call Trace: [93070.253716] dump_stack+0x41/0x60 [93070.254682] warn_alloc.cold.127+0x7b/0x108 [93070.255796] __alloc_pages_slowpath+0xcb2/0xcd0 [93070.256984] ? __alloc_pages_nodemask+0x166/0x330 [93070.258263] __alloc_pages_nodemask+0x2e2/0x330 [93070.259476] kmalloc_order+0x28/0x90 [93070.260481] kmalloc_order_trace+0x1d/0xb0 [93070.261577] __kmalloc+0x203/0x250 [93070.262529] virtqueue_add+0x493/0xc70 [93070.263564] ? finish_wait+0x80/0x80 [93070.264585] virtqueue_add_sgs+0x80/0xa0 [93070.265628] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93070.266965] ? scsi_alloc_sgtables+0x84/0x1c0 [93070.268108] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93070.269405] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93070.270876] scsi_queue_rq+0x512/0xb10 [93070.271920] __blk_mq_try_issue_directly+0x163/0x200 [93070.273195] blk_mq_request_issue_directly+0x4e/0xb0 [93070.274464] blk_mq_try_issue_list_directly+0x62/0x100 [93070.275765] blk_mq_sched_insert_requests+0xa4/0xf0 [93070.277023] blk_mq_flush_plug_list+0x135/0x220 [93070.278232] blk_flush_plug_list+0xd7/0x100 [93070.279379] blk_finish_plug+0x25/0x36 [93070.280407] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93070.281902] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93070.283261] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93070.284821] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93070.286362] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93070.287707] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93070.288991] ofd_commitrw+0x5b4/0xd20 [ofd] [93070.290169] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93070.291604] obd_commitrw+0x1b6/0x370 [ptlrpc] [93070.292914] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93070.294273] ? newidle_balance+0x2b6/0x3b0 [93070.295388] ? internal_add_timer+0x42/0x70 [93070.296489] ? _cond_resched+0x15/0x30 [93070.297518] ? mutex_lock+0xe/0x30 [93070.298488] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93070.299935] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93070.301468] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93070.303055] ? finish_wait+0x80/0x80 [93070.304100] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93070.305420] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93070.306845] kthread+0x134/0x150 [93070.307762] ? set_kthread_struct+0x50/0x50 [93070.308849] ret_from_fork+0x1f/0x40 [93070.309839] CPU: 5 PID: 527021 Comm: ll_ost_io01_008 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93070.312990] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93070.315070] Call Trace: [93070.315821] dump_stack+0x41/0x60 [93070.316737] warn_alloc.cold.127+0x7b/0x108 [93070.317843] __alloc_pages_slowpath+0xcb2/0xcd0 [93070.319009] ? __alloc_pages_nodemask+0x166/0x330 [93070.320210] __alloc_pages_nodemask+0x2e2/0x330 [93070.321365] kmalloc_order+0x28/0x90 [93070.322337] kmalloc_order_trace+0x1d/0xb0 [93070.323409] __kmalloc+0x203/0x250 [93070.324337] ? __wake_up_common+0x7a/0x190 [93070.325418] virtqueue_add+0x493/0xc70 [93070.326435] ? finish_wait+0x80/0x80 [93070.327415] virtqueue_add_sgs+0x80/0xa0 [93070.328461] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93070.329785] ? scsi_alloc_sgtables+0x84/0x1c0 [93070.330899] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93070.332135] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93070.333524] scsi_queue_rq+0x512/0xb10 [93070.334560] __blk_mq_try_issue_directly+0x163/0x200 [93070.335850] blk_mq_request_issue_directly+0x4e/0xb0 [93070.337095] blk_mq_try_issue_list_directly+0x62/0x100 [93070.338355] blk_mq_sched_insert_requests+0xa4/0xf0 [93070.339605] blk_mq_flush_plug_list+0x135/0x220 [93070.340798] blk_flush_plug_list+0xd7/0x100 [93070.341948] blk_finish_plug+0x25/0x36 [93070.342970] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93070.344404] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93070.345733] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93070.347251] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93070.348730] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93070.350041] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93070.351263] ofd_commitrw+0x5b4/0xd20 [ofd] [93070.352385] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93070.353697] obd_commitrw+0x1b6/0x370 [ptlrpc] [93070.354996] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93070.356333] ? flush_work+0x42/0x1d0 [93070.357312] ? internal_add_timer+0x42/0x70 [93070.358420] ? _cond_resched+0x15/0x30 [93070.359426] ? mutex_lock+0xe/0x30 [93070.360363] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93070.361772] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93070.363284] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93070.364859] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93070.366149] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93070.367531] kthread+0x134/0x150 [93070.368446] ? set_kthread_struct+0x50/0x50 [93070.369563] ret_from_fork+0x1f/0x40 [93932.261820] ll_ost_io02_019: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [93932.265272] CPU: 8 PID: 527172 Comm: ll_ost_io02_019 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93932.265805] ll_ost_io04_056: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [93932.268203] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93932.268206] Call Trace: [93932.270743] / [93932.272944] dump_stack+0x41/0x60 [93932.273779] ,mems_allowed=0 [93932.274494] warn_alloc.cold.127+0x7b/0x108 [93932.276425] __alloc_pages_slowpath+0xcb2/0xcd0 [93932.279456] ? __alloc_pages_nodemask+0x166/0x330 [93932.280718] __alloc_pages_nodemask+0x2e2/0x330 [93932.281925] kmalloc_order+0x28/0x90 [93932.282963] kmalloc_order_trace+0x1d/0xb0 [93932.284067] __kmalloc+0x203/0x250 [93932.285038] virtqueue_add+0x493/0xc70 [93932.286058] ? finish_wait+0x80/0x80 [93932.287059] virtqueue_add_sgs+0x80/0xa0 [93932.288165] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93932.289566] ? scsi_alloc_sgtables+0x84/0x1c0 [93932.290722] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93932.292027] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93932.293453] scsi_queue_rq+0x512/0xb10 [93932.294477] __blk_mq_try_issue_directly+0x163/0x200 [93932.295734] blk_mq_request_issue_directly+0x4e/0xb0 [93932.297011] blk_mq_try_issue_list_directly+0x62/0x100 [93932.298323] blk_mq_sched_insert_requests+0xa4/0xf0 [93932.299618] blk_mq_flush_plug_list+0x135/0x220 [93932.300816] blk_flush_plug_list+0xd7/0x100 [93932.301941] blk_finish_plug+0x25/0x36 [93932.302991] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93932.304580] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [93932.306020] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93932.307613] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93932.309180] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93932.310570] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93932.311907] ofd_commitrw+0x5b4/0xd20 [ofd] [93932.313034] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93932.314406] obd_commitrw+0x1b6/0x370 [ptlrpc] [93932.315691] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93932.317034] ? flush_work+0x42/0x1d0 [93932.318066] ? internal_add_timer+0x42/0x70 [93932.319204] ? _cond_resched+0x15/0x30 [93932.320244] ? mutex_lock+0xe/0x30 [93932.321207] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93932.322638] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93932.324216] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93932.325781] ? ofd_object_print+0x20/0x20 [ofd] [93932.327011] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93932.328301] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93932.329670] kthread+0x134/0x150 [93932.330583] ? set_kthread_struct+0x50/0x50 [93932.331705] ret_from_fork+0x1f/0x40 [93932.332703] warn_alloc_show_mem: 6 callbacks suppressed [93932.332705] Mem-Info: [93932.332703] CPU: 16 PID: 527693 Comm: ll_ost_io04_056 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [93932.334004] active_anon:27731 inactive_anon:309196 isolated_anon:0 active_file:29866695 inactive_file:5377108 isolated_file:0 unevictable:0 dirty:705 writeback:0 slab_reclaimable:841225 slab_unreclaimable:619870 mapped:21353 shmem:148 pagetables:5580 bounce:0 free:973402 free_pcp:436 free_cma:0 [93932.334796] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [93932.337659] Node 0 active_anon:110924kB inactive_anon:1236784kB active_file:119471144kB inactive_file:21503200kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:85412kB dirty:2820kB writeback:0kB shmem:592kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35280kB pagetables:22320kB all_unreclaimable? no [93932.345615] Call Trace: [93932.347767] Node 0 [93932.354590] dump_stack+0x41/0x60 [93932.355458] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [93932.356225] warn_alloc.cold.127+0x7b/0x108 [93932.357228] lowmem_reserve[]: [93932.362172] __alloc_pages_slowpath+0xcb2/0xcd0 [93932.363374] 0 [93932.364348] ? __alloc_pages_nodemask+0x166/0x330 [93932.365650] 913 [93932.366419] __alloc_pages_nodemask+0x2e2/0x330 [93932.367747] 150007 [93932.368519] kmalloc_order+0x28/0x90 [93932.369828] 150007 [93932.370646] kmalloc_order_trace+0x1d/0xb0 [93932.371775] 150007 [93932.372589] __kmalloc+0x203/0x250 [93932.374619] virtqueue_add+0x493/0xc70 [93932.375719] Node 0 [93932.376456] ? finish_wait+0x80/0x80 [93932.377605] DMA32 free:596652kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:1512kB active_file:1328kB inactive_file:280kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [93932.378421] virtqueue_add_sgs+0x80/0xa0 [93932.379534] lowmem_reserve[]: [93932.385160] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [93932.386380] 0 [93932.387414] ? scsi_alloc_sgtables+0x84/0x1c0 [93932.388951] 0 [93932.389741] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [93932.391074] 149093 [93932.391842] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [93932.393288] 149093 [93932.394145] scsi_queue_rq+0x512/0xb10 [93932.395704] 149093 [93932.396550] __blk_mq_try_issue_directly+0x163/0x200 [93932.398736] blk_mq_request_issue_directly+0x4e/0xb0 [93932.400150] Node 0 [93932.400917] blk_mq_try_issue_list_directly+0x62/0x100 [93932.402314] Normal free:3291604kB min:49268kB low:201940kB high:354612kB active_anon:110544kB inactive_anon:1235232kB active_file:119473712kB inactive_file:21491844kB unevictable:0kB writepending:2864kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1996kB local_pcp:24kB free_cma:0kB [93932.403173] blk_mq_sched_insert_requests+0xa4/0xf0 [93932.404593] lowmem_reserve[]: [93932.410876] blk_mq_flush_plug_list+0x135/0x220 [93932.412215] 0 [93932.413205] blk_flush_plug_list+0xd7/0x100 [93932.414467] 0 [93932.415185] blk_finish_plug+0x25/0x36 [93932.416374] 0 [93932.417095] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [93932.418188] 0 [93932.418913] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93932.420418] 0 [93932.421160] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [93932.423584] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [93932.425220] Node 0 [93932.425946] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [93932.427368] DMA: [93932.428249] ofd_commitrw+0x5b4/0xd20 [ofd] [93932.429598] 0*4kB [93932.430401] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [93932.431614] 0*8kB [93932.432443] obd_commitrw+0x1b6/0x370 [ptlrpc] [93932.433749] 0*16kB [93932.434556] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [93932.435806] 0*32kB [93932.436637] ? set_next_entity+0xb5/0x1e0 [93932.437985] 0*64kB [93932.438839] ? internal_add_timer+0x42/0x70 [93932.440067] 0*128kB [93932.440924] ? _cond_resched+0x15/0x30 [93932.442127] 0*256kB [93932.443070] ? mutex_lock+0xe/0x30 [93932.444237] 0*512kB [93932.445048] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [93932.446085] 1*1024kB [93932.446906] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [93932.448265] (U) [93932.449081] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [93932.450575] 1*2048kB [93932.451325] ? finish_wait+0x80/0x80 [93932.452827] (M) [93932.453695] ptlrpc_main+0xbec/0x1530 [ptlrpc] [93932.454734] 2*4096kB [93932.455454] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [93932.456693] (M) [93932.457527] kthread+0x134/0x150 [93932.458904] = 11264kB [93932.459640] ? set_kthread_struct+0x50/0x50 [93932.460657] Node 0 [93932.461468] ret_from_fork+0x1f/0x40 [93932.462652] DMA32: 117*4kB (UMH) 89*8kB (UMH) 81*16kB (UMEH) 96*32kB (UMEH) 101*64kB (UMH) 119*128kB (UMH) 105*256kB (UMH) 97*512kB (UM) 78*1024kB (UMH) 56*2048kB (UM) 73*4096kB (UM) = 597356kB [93932.469294] Node 0 Normal: 559503*4kB (UME) 118019*8kB (UME) 5282*16kB (UMEH) 11*32kB (UH) 1*64kB (H) 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 3267092kB [93932.473504] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [93932.475706] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [93932.477703] 35249498 total pagecache pages [93932.478846] 1631 pages in swap cache [93932.479882] Swap cache stats: add 1408253, delete 1406640, find 66913/80548 [93932.481585] Free swap = 5991268kB [93932.482598] Total swap = 11075580kB [93932.483617] 39321462 pages RAM [93932.484586] 0 pages HighMem/MovableOnly [93932.485661] 905848 pages reserved [93932.486623] 0 pages hwpoisoned [94028.755285] ll_ost_io04_075: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94028.757515] ll_ost_io04_033: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94028.758492] CPU: 19 PID: 528063 Comm: ll_ost_io04_075 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94028.758813] ll_ost_io02_035: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94028.765807] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94028.767650] Call Trace: [94028.768388] dump_stack+0x41/0x60 [94028.769207] warn_alloc.cold.127+0x7b/0x108 [94028.770161] __alloc_pages_slowpath+0xcb2/0xcd0 [94028.771156] ? __alloc_pages_nodemask+0x166/0x330 [94028.772214] __alloc_pages_nodemask+0x2e2/0x330 [94028.773244] kmalloc_order+0x28/0x90 [94028.774103] kmalloc_order_trace+0x1d/0xb0 [94028.775058] __kmalloc+0x203/0x250 [94028.775895] virtqueue_add+0x493/0xc70 [94028.776781] ? finish_wait+0x80/0x80 [94028.777687] virtqueue_add_sgs+0x80/0xa0 [94028.778612] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94028.779779] ? scsi_alloc_sgtables+0x84/0x1c0 [94028.780765] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94028.781869] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94028.783055] scsi_queue_rq+0x512/0xb10 [94028.783924] __blk_mq_try_issue_directly+0x163/0x200 [94028.785007] blk_mq_request_issue_directly+0x4e/0xb0 [94028.786093] blk_mq_try_issue_list_directly+0x62/0x100 [94028.786295] ll_ost_io02_027: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [94028.787218] blk_mq_sched_insert_requests+0xa4/0xf0 [94028.787221] ,mems_allowed=0 [94028.789367] blk_mq_flush_plug_list+0x135/0x220 [94028.792630] blk_flush_plug_list+0xd7/0x100 [94028.793604] blk_finish_plug+0x25/0x36 [94028.794497] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94028.795753] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94028.796911] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94028.798250] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94028.799573] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94028.800714] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94028.801791] ofd_commitrw+0x5b4/0xd20 [ofd] [94028.802785] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94028.803924] obd_commitrw+0x1b6/0x370 [ptlrpc] [94028.805017] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94028.806142] ? flush_work+0x42/0x1d0 [94028.806969] ? internal_add_timer+0x42/0x70 [94028.807885] ? _cond_resched+0x15/0x30 [94028.808754] ? mutex_lock+0xe/0x30 [94028.809568] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94028.810751] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94028.812055] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94028.813379] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94028.814492] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94028.815677] kthread+0x134/0x150 [94028.816462] ? set_kthread_struct+0x50/0x50 [94028.817401] ret_from_fork+0x1f/0x40 [94028.818240] warn_alloc_show_mem: 1 callbacks suppressed [94028.818241] CPU: 10 PID: 527184 Comm: ll_ost_io02_027 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94028.818249] Mem-Info: [94028.819722] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94028.822163] active_anon:27756 inactive_anon:308721 isolated_anon:0 active_file:30881951 inactive_file:4713528 isolated_file:32 unevictable:0 dirty:438 writeback:0 slab_reclaimable:846381 slab_unreclaimable:620619 mapped:20130 shmem:148 pagetables:5585 bounce:0 free:614914 free_pcp:507 free_cma:0 [94028.823074] Call Trace: [94028.824884] Node 0 active_anon:111024kB inactive_anon:1234884kB active_file:123546680kB inactive_file:18849528kB unevictable:0kB isolated(anon):0kB isolated(file):128kB mapped:80520kB dirty:1752kB writeback:0kB shmem:592kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35296kB pagetables:22340kB all_unreclaimable? no [94028.833297] dump_stack+0x41/0x60 [94028.833993] Node 0 [94028.841227] warn_alloc.cold.127+0x7b/0x108 [94028.842068] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94028.842893] __alloc_pages_slowpath+0xcb2/0xcd0 [94028.843882] lowmem_reserve[]: [94028.849067] __alloc_pages_nodemask+0x2e2/0x330 [94028.850148] 0 [94028.851141] kmalloc_order+0x28/0x90 [94028.852191] 913 [94028.852947] kmalloc_order_trace+0x1d/0xb0 [94028.853849] 150007 [94028.854621] __kmalloc+0x203/0x250 [94028.855622] 150007 [94028.856356] virtqueue_add+0x493/0xc70 [94028.857226] 150007 [94028.858050] virtqueue_add_sgs+0x80/0xa0 [94028.859813] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94028.860775] Node 0 [94028.861457] ? scsi_alloc_sgtables+0x84/0x1c0 [94028.862673] DMA32 free:596736kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:1488kB active_file:504kB inactive_file:84kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94028.863453] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94028.864470] lowmem_reserve[]: 0 [94028.869242] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94028.870392] 0 [94028.871383] scsi_queue_rq+0x512/0xb10 [94028.872637] 149093 [94028.873230] __blk_mq_try_issue_directly+0x163/0x200 [94028.874210] 149093 [94028.874857] blk_mq_request_issue_directly+0x4e/0xb0 [94028.875993] 149093 [94028.876631] blk_mq_try_issue_list_directly+0x62/0x100 [94028.878506] blk_mq_sched_insert_requests+0xa4/0xf0 [94028.879684] Node 0 [94028.880399] blk_mq_flush_plug_list+0x135/0x220 [94028.881556] Normal free:1893524kB min:49268kB low:201940kB high:354612kB active_anon:110644kB inactive_anon:1233396kB active_file:123498776kB inactive_file:18840480kB unevictable:0kB writepending:1708kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1828kB local_pcp:84kB free_cma:0kB [94028.882389] blk_flush_plug_list+0xd7/0x100 [94028.883459] lowmem_reserve[]: 0 [94028.888961] blk_finish_plug+0x25/0x36 [94028.889978] 0 [94028.890809] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94028.891732] 0 [94028.892465] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [94028.893707] 0 [94028.894286] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94028.895439] 0 [94028.896132] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94028.898180] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94028.899502] Node 0 [94028.900184] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94028.901354] DMA: [94028.902006] ofd_commitrw+0x5b4/0xd20 [ofd] [94028.903095] 0*4kB [94028.903665] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94028.904605] 0*8kB [94028.905294] obd_commitrw+0x1b6/0x370 [ptlrpc] [94028.906320] 0*16kB [94028.907011] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94028.908001] 0*32kB [94028.908820] ? flush_work+0x42/0x1d0 [94028.909834] 0*64kB [94028.910588] ? internal_add_timer+0x42/0x70 [94028.912244] 0*128kB [94028.912818] ? _cond_resched+0x15/0x30 [94028.913831] 0*256kB [94028.914395] ? mutex_lock+0xe/0x30 [94028.915343] 0*512kB [94028.915928] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94028.916764] 1*1024kB [94028.917337] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94028.918437] (U) [94028.919011] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94028.920227] 1*2048kB [94028.920747] ? finish_wait+0x80/0x80 [94028.922013] (M) [94028.922599] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94028.923462] 2*4096kB [94028.924008] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94028.925000] (M) [94028.925635] kthread+0x134/0x150 [94028.926733] = 11264kB [94028.927296] ? set_kthread_struct+0x50/0x50 [94028.928106] Node 0 [94028.928731] ret_from_fork+0x1f/0x40 [94028.929713] DMA32: [94028.930368] CPU: 8 PID: 527211 Comm: ll_ost_io02_035 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94028.931243] 110*4kB [94028.931998] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94028.934417] (UMH) [94028.935166] Call Trace: [94028.936985] 111*8kB [94028.937652] dump_stack+0x41/0x60 [94028.938332] (UMH) [94028.939065] warn_alloc.cold.127+0x7b/0x108 [94028.939891] 97*16kB [94028.940490] __alloc_pages_slowpath+0xcb2/0xcd0 [94028.941465] (UMEH) 77*32kB (UMH) 104*64kB (UMEH) 117*128kB [94028.942093] __alloc_pages_nodemask+0x2e2/0x330 [94028.943123] (UM) [94028.944243] kmalloc_order+0x28/0x90 [94028.945268] 106*256kB [94028.945943] kmalloc_order_trace+0x1d/0xb0 [94028.946836] (UME) [94028.947690] __kmalloc+0x203/0x250 [94028.948650] 98*512kB [94028.949416] virtqueue_add+0x493/0xc70 [94028.950265] (UMH) [94028.951068] ? finish_wait+0x80/0x80 [94028.951963] 79*1024kB [94028.952673] virtqueue_add_sgs+0x80/0xa0 [94028.953559] (UMEH) [94028.954412] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94028.955352] 57*2048kB [94028.956174] ? scsi_alloc_sgtables+0x84/0x1c0 [94028.957334] (UME) [94028.958194] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94028.959181] 72*4096kB [94028.959903] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94028.960991] (UM) [94028.961842] scsi_queue_rq+0x512/0xb10 [94028.963057] = 596832kB [94028.963736] __blk_mq_try_issue_directly+0x163/0x200 [94028.964643] Node 0 [94028.965459] blk_mq_request_issue_directly+0x4e/0xb0 [94028.966559] Normal: [94028.967333] blk_mq_try_issue_list_directly+0x62/0x100 [94028.968399] 114327*4kB [94028.969216] blk_mq_sched_insert_requests+0xa4/0xf0 [94028.970311] (UME) 164357*8kB [94028.971166] blk_mq_flush_plug_list+0x135/0x220 [94028.972255] (UME) [94028.973011] blk_flush_plug_list+0xd7/0x100 [94028.974050] 7716*16kB [94028.974833] blk_finish_plug+0x25/0x36 [94028.975836] (UMEH) [94028.976691] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94028.977586] 0*32kB [94028.978394] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94028.979588] 0*64kB [94028.980372] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94028.981686] 0*128kB [94028.982426] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94028.983678] 0*256kB [94028.984360] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94028.985490] 0*512kB [94028.986147] ofd_commitrw+0x5b4/0xd20 [ofd] [94028.987217] 0*1024kB [94028.987832] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94028.988756] 0*2048kB [94028.989482] obd_commitrw+0x1b6/0x370 [ptlrpc] [94028.990492] 0*4096kB [94028.991238] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94028.992214] = 1895620kB [94028.993031] ? common_interrupt+0xa/0xf [94028.994082] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [94028.994820] ? _cond_resched+0x15/0x30 [94028.995727] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [94028.997553] ? mutex_lock+0xe/0x30 [94028.998460] 35586161 total pagecache pages [94029.000528] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94029.001337] 1636 pages in swap cache [94029.002283] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94029.003350] Swap cache stats: add 1411096, delete 1409478, find 67298/81033 [94029.004430] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94029.005629] Free swap = 5983556kB [94029.007349] ? finish_wait+0x80/0x80 [94029.008591] Total swap = 11075580kB [94029.009607] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94029.010459] 39321462 pages RAM [94029.011506] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94029.012476] 0 pages HighMem/MovableOnly [94029.013303] kthread+0x134/0x150 [94029.014419] 905848 pages reserved [94029.015534] ? set_kthread_struct+0x50/0x50 [94029.016349] 0 pages hwpoisoned [94029.017308] ret_from_fork+0x1f/0x40 [94029.019848] CPU: 18 PID: 527279 Comm: ll_ost_io04_033 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94029.023163] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94029.025441] Call Trace: [94029.026277] dump_stack+0x41/0x60 [94029.027245] warn_alloc.cold.127+0x7b/0x108 [94029.028389] __alloc_pages_slowpath+0xcb2/0xcd0 [94029.029617] ? __alloc_pages_nodemask+0x166/0x330 [94029.030941] __alloc_pages_nodemask+0x2e2/0x330 [94029.032172] kmalloc_order+0x28/0x90 [94029.033296] kmalloc_order_trace+0x1d/0xb0 [94029.034501] __kmalloc+0x203/0x250 [94029.035525] virtqueue_add+0x493/0xc70 [94029.036581] ? finish_wait+0x80/0x80 [94029.037601] virtqueue_add_sgs+0x80/0xa0 [94029.038671] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94029.040063] ? scsi_alloc_sgtables+0x84/0x1c0 [94029.041248] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94029.042583] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94029.044057] scsi_queue_rq+0x512/0xb10 [94029.045094] __blk_mq_try_issue_directly+0x163/0x200 [94029.046425] blk_mq_request_issue_directly+0x4e/0xb0 [94029.047707] blk_mq_try_issue_list_directly+0x62/0x100 [94029.049056] blk_mq_sched_insert_requests+0xa4/0xf0 [94029.050328] blk_mq_flush_plug_list+0x135/0x220 [94029.051531] blk_flush_plug_list+0xd7/0x100 [94029.052664] blk_finish_plug+0x25/0x36 [94029.053719] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94029.055218] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94029.056665] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94029.058286] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94029.059871] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94029.061248] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94029.062583] ofd_commitrw+0x5b4/0xd20 [ofd] [94029.063742] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94029.065138] obd_commitrw+0x1b6/0x370 [ptlrpc] [94029.066479] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94029.067840] ? newidle_balance+0x2b6/0x3b0 [94029.068964] ? internal_add_timer+0x42/0x70 [94029.070103] ? _cond_resched+0x15/0x30 [94029.071138] ? mutex_lock+0xe/0x30 [94029.072092] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94029.073489] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94029.075036] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94029.076594] ? finish_wait+0x80/0x80 [94029.077576] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94029.078883] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94029.080248] kthread+0x134/0x150 [94029.081154] ? set_kthread_struct+0x50/0x50 [94029.082244] ret_from_fork+0x1f/0x40 [94035.445193] ll_ost_io01_043: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [94035.445196] ll_ost_io02_042: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [94035.445201] ll_ost_io02_070: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94035.445213] CPU: 11 PID: 527735 Comm: ll_ost_io02_070 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.445218] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.445219] Call Trace: [94035.445227] ll_ost_io01_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [94035.445228] ll_ost_io03_080: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [94035.445225] dump_stack+0x41/0x60 [94035.445233] ,cpuset= [94035.445234] ,cpuset= [94035.445235] /,mems_allowed=0 [94035.445237] / [94035.445238] ,mems_allowed=0 [94035.445235] warn_alloc.cold.127+0x7b/0x108 [94035.445244] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.445248] ? __alloc_pages_nodemask+0x166/0x330 [94035.445252] __alloc_pages_nodemask+0x2e2/0x330 [94035.445255] kmalloc_order+0x28/0x90 [94035.445263] kmalloc_order_trace+0x1d/0xb0 [94035.445267] __kmalloc+0x203/0x250 [94035.445273] virtqueue_add+0x493/0xc70 [94035.445280] ? finish_wait+0x80/0x80 [94035.445286] virtqueue_add_sgs+0x80/0xa0 [94035.445292] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.445298] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.445305] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.445309] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.445313] scsi_queue_rq+0x512/0xb10 [94035.445317] __blk_mq_try_issue_directly+0x163/0x200 [94035.445325] blk_mq_request_issue_directly+0x4e/0xb0 [94035.445330] blk_mq_try_issue_list_directly+0x62/0x100 [94035.445334] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.445339] blk_mq_flush_plug_list+0x135/0x220 [94035.445343] blk_flush_plug_list+0xd7/0x100 [94035.445349] blk_finish_plug+0x25/0x36 [94035.445355] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.445395] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.445427] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.445455] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.445482] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.445511] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.445535] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.445626] ll_ost_io02_081: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94035.445559] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.445695] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.445812] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.445936] ? flush_work+0x42/0x1d0 [94035.445942] ? internal_add_timer+0x42/0x70 [94035.445947] ? _cond_resched+0x15/0x30 [94035.445951] ? mutex_lock+0xe/0x30 [94035.445954] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.446080] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.446194] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.446303] ? finish_wait+0x80/0x80 [94035.446307] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.446430] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.446538] kthread+0x134/0x150 [94035.446542] ? set_kthread_struct+0x50/0x50 [94035.446545] ret_from_fork+0x1f/0x40 [94035.446551] warn_alloc_show_mem: 3 callbacks suppressed [94035.446553] Mem-Info: [94035.446552] CPU: 8 PID: 527874 Comm: ll_ost_io02_081 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.446555] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.446557] Call Trace: [94035.446556] active_anon:27724 inactive_anon:307878 isolated_anon:0 active_file:30819888 inactive_file:4201384 isolated_file:31 unevictable:0 dirty:1876 writeback:0 slab_reclaimable:846730 slab_unreclaimable:621665 mapped:20363 shmem:148 pagetables:5585 bounce:0 free:1187441 free_pcp:417 free_cma:0 [94035.446562] Node 0 active_anon:110896kB inactive_anon:1231512kB active_file:123279552kB inactive_file:16805536kB unevictable:0kB isolated(anon):0kB isolated(file):124kB mapped:81452kB dirty:7504kB writeback:0kB shmem:592kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35280kB pagetables:22340kB all_unreclaimable? no [94035.446560] dump_stack+0x41/0x60 [94035.446569] Node 0 [94035.446568] warn_alloc.cold.127+0x7b/0x108 [94035.446570] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94035.446573] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.446576] lowmem_reserve[]: 0 [94035.446577] ? __alloc_pages_nodemask+0x166/0x330 [94035.446578] 913 150007 150007 150007 [94035.446580] __alloc_pages_nodemask+0x2e2/0x330 [94035.446583] Node 0 [94035.446584] kmalloc_order+0x28/0x90 [94035.446585] DMA32 free:596420kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:1488kB active_file:504kB inactive_file:80kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94035.446589] kmalloc_order_trace+0x1d/0xb0 [94035.446591] lowmem_reserve[]: 0 0 149093 149093 149093 [94035.446593] __kmalloc+0x203/0x250 [94035.446597] Node 0 [94035.446597] ? __wake_up_common+0x7a/0x190 [94035.446598] Normal free:4142220kB min:49268kB low:201940kB high:354612kB active_anon:110516kB inactive_anon:1230024kB active_file:123279856kB inactive_file:16805448kB unevictable:0kB writepending:7772kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1624kB local_pcp:96kB free_cma:0kB [94035.446601] virtqueue_add+0x493/0xc70 [94035.446604] lowmem_reserve[]: 0 0 0 [94035.446606] ? finish_wait+0x80/0x80 [94035.446607] 0 0 [94035.446610] Node 0 [94035.446609] virtqueue_add_sgs+0x80/0xa0 [94035.446611] DMA: 0*4kB 0*8kB 0*16kB 0*32kB [94035.446614] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.446617] 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB [94035.446620] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.446623] (U) 1*2048kB (M) 2*4096kB [94035.446625] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.446627] (M) = 11264kB [94035.446630] Node 0 [94035.446629] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.446630] DMA32: 111*4kB (UMH) [94035.446633] scsi_queue_rq+0x512/0xb10 [94035.446634] 111*8kB (UMH) 97*16kB (UMEH) 95*32kB [94035.446636] __blk_mq_try_issue_directly+0x163/0x200 [94035.446639] (UMH) 103*64kB (UMEH) 117*128kB [94035.446642] blk_mq_request_issue_directly+0x4e/0xb0 [94035.446644] (UM) 106*256kB (UME) 98*512kB [94035.446646] blk_mq_try_issue_list_directly+0x62/0x100 [94035.446648] (UMH) 78*1024kB (UME) 57*2048kB (UME) [94035.446650] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.446653] 72*4096kB (UM) = 596324kB [94035.446654] blk_mq_flush_plug_list+0x135/0x220 [94035.446656] Node 0 Normal: 419340*4kB (UME) 283047*8kB (UME) [94035.446659] blk_flush_plug_list+0xd7/0x100 [94035.446661] 12480*16kB (UMH) 0*32kB 0*64kB [94035.446664] blk_finish_plug+0x25/0x36 [94035.446666] 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 4141416kB [94035.446673] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [94035.446676] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [94035.446678] 35021240 total pagecache pages [94035.446679] 1632 pages in swap cache [94035.446680] Swap cache stats: add 1411906, delete 1410292, find 67395/81148 [94035.446682] Free swap = 5981244kB [94035.446683] Total swap = 11075580kB [94035.446684] 39321462 pages RAM [94035.446685] 0 pages HighMem/MovableOnly [94035.446686] 905848 pages reserved [94035.446686] 0 pages hwpoisoned [94035.446669] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.446703] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.446733] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.446758] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.446784] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.446810] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.446829] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.446846] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.446962] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.447077] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.447190] ? flush_work+0x42/0x1d0 [94035.447195] ? internal_add_timer+0x42/0x70 [94035.447198] ? _cond_resched+0x15/0x30 [94035.447201] ? mutex_lock+0xe/0x30 [94035.447203] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.447315] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.447425] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.447526] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.447634] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.447732] kthread+0x134/0x150 [94035.447736] ? set_kthread_struct+0x50/0x50 [94035.447739] ret_from_fork+0x1f/0x40 [94035.447745] CPU: 7 PID: 527199 Comm: ll_ost_io01_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.447750] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.447751] Call Trace: [94035.447755] dump_stack+0x41/0x60 [94035.447763] warn_alloc.cold.127+0x7b/0x108 [94035.447770] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.447775] ? __alloc_pages_nodemask+0x166/0x330 [94035.447778] __alloc_pages_nodemask+0x2e2/0x330 [94035.447781] kmalloc_order+0x28/0x90 [94035.447787] kmalloc_order_trace+0x1d/0xb0 [94035.447791] __kmalloc+0x203/0x250 [94035.447796] virtqueue_add+0x493/0xc70 [94035.447802] virtqueue_add_sgs+0x80/0xa0 [94035.447805] ,cpuset= [94035.447807] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.447813] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.447819] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.447823] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.447827] scsi_queue_rq+0x512/0xb10 [94035.447830] __blk_mq_try_issue_directly+0x163/0x200 [94035.447837] blk_mq_request_issue_directly+0x4e/0xb0 [94035.447841] blk_mq_try_issue_list_directly+0x62/0x100 [94035.447845] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.447849] blk_mq_flush_plug_list+0x135/0x220 [94035.447854] blk_flush_plug_list+0xd7/0x100 [94035.447860] blk_finish_plug+0x25/0x36 [94035.447865] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.447901] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.447936] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.447966] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.447996] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.448025] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.448045] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.448064] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.448194] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.448314] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.448427] ? newidle_balance+0x2b6/0x3b0 [94035.448432] ? internal_add_timer+0x42/0x70 [94035.448436] ? _cond_resched+0x15/0x30 [94035.448438] ? mutex_lock+0xe/0x30 [94035.448442] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.448571] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.448675] ll_ost_io02_068: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94035.448699] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.448770] ll_ost_io00_052: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94035.448824] ? finish_wait+0x80/0x80 [94035.448828] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.448950] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.449074] kthread+0x134/0x150 [94035.449078] ? set_kthread_struct+0x50/0x50 [94035.449082] ret_from_fork+0x1f/0x40 [94035.449089] CPU: 12 PID: 527553 Comm: ll_ost_io03_080 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.449094] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.449095] Call Trace: [94035.449098] dump_stack+0x41/0x60 [94035.449105] warn_alloc.cold.127+0x7b/0x108 [94035.449111] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.449115] ? __alloc_pages_nodemask+0x166/0x330 [94035.449119] __alloc_pages_nodemask+0x2e2/0x330 [94035.449122] kmalloc_order+0x28/0x90 [94035.449128] kmalloc_order_trace+0x1d/0xb0 [94035.449131] __kmalloc+0x203/0x250 [94035.449136] virtqueue_add+0x493/0xc70 [94035.449143] ? finish_wait+0x80/0x80 [94035.449146] virtqueue_add_sgs+0x80/0xa0 [94035.449152] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.449158] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.449163] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.449167] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.449171] scsi_queue_rq+0x512/0xb10 [94035.449174] __blk_mq_try_issue_directly+0x163/0x200 [94035.449181] blk_mq_request_issue_directly+0x4e/0xb0 [94035.449185] blk_mq_try_issue_list_directly+0x62/0x100 [94035.449189] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.449193] blk_mq_flush_plug_list+0x135/0x220 [94035.449198] blk_flush_plug_list+0xd7/0x100 [94035.449203] blk_finish_plug+0x25/0x36 [94035.449208] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.449242] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.449272] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.449301] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.449330] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.449359] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.449380] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.449399] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.449524] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.449650] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.449763] ? flush_work+0x42/0x1d0 [94035.449768] ? internal_add_timer+0x42/0x70 [94035.449771] ? _cond_resched+0x15/0x30 [94035.449774] ? mutex_lock+0xe/0x30 [94035.449777] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.449892] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.449997] ll_ost_io04_007: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94035.450007] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.450124] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.450276] / [94035.450248] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.450370] kthread+0x134/0x150 [94035.450374] ? set_kthread_struct+0x50/0x50 [94035.450377] ret_from_fork+0x1f/0x40 [94035.450383] CPU: 19 PID: 527171 Comm: ll_ost_io04_007 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.450388] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.450389] Call Trace: [94035.450392] dump_stack+0x41/0x60 [94035.450399] warn_alloc.cold.127+0x7b/0x108 [94035.450404] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.450408] ? __alloc_pages_nodemask+0x166/0x330 [94035.450412] __alloc_pages_nodemask+0x2e2/0x330 [94035.450415] kmalloc_order+0x28/0x90 [94035.450421] kmalloc_order_trace+0x1d/0xb0 [94035.450425] __kmalloc+0x203/0x250 [94035.450430] virtqueue_add+0x493/0xc70 [94035.450436] virtqueue_add_sgs+0x80/0xa0 [94035.450441] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.450447] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.450452] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.450456] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.450460] scsi_queue_rq+0x512/0xb10 [94035.450463] __blk_mq_try_issue_directly+0x163/0x200 [94035.450469] blk_mq_request_issue_directly+0x4e/0xb0 [94035.450473] blk_mq_try_issue_list_directly+0x62/0x100 [94035.450477] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.450482] blk_mq_flush_plug_list+0x135/0x220 [94035.450486] blk_flush_plug_list+0xd7/0x100 [94035.450491] blk_finish_plug+0x25/0x36 [94035.450496] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.450530] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.450562] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.450589] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.450618] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.450648] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.450669] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.450689] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.450809] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.450924] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.451041] ? flush_work+0x42/0x1d0 [94035.451046] ? internal_add_timer+0x42/0x70 [94035.451049] ? _cond_resched+0x15/0x30 [94035.451052] ? mutex_lock+0xe/0x30 [94035.451056] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.451175] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.451288] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.451408] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.451536] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.451647] kthread+0x134/0x150 [94035.451651] ? set_kthread_struct+0x50/0x50 [94035.451655] ret_from_fork+0x1f/0x40 [94035.451661] CPU: 8 PID: 527733 Comm: ll_ost_io02_068 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.451665] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.451666] Call Trace: [94035.451668] dump_stack+0x41/0x60 [94035.451673] warn_alloc.cold.127+0x7b/0x108 [94035.451678] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.451682] ? __alloc_pages_nodemask+0x166/0x330 [94035.451685] __alloc_pages_nodemask+0x2e2/0x330 [94035.451687] kmalloc_order+0x28/0x90 [94035.451692] kmalloc_order_trace+0x1d/0xb0 [94035.451695] __kmalloc+0x203/0x250 [94035.451700] virtqueue_add+0x493/0xc70 [94035.451704] ? finish_wait+0x80/0x80 [94035.451707] virtqueue_add_sgs+0x80/0xa0 [94035.451711] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.451716] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.451720] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.451723] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.451726] scsi_queue_rq+0x512/0xb10 [94035.451729] __blk_mq_try_issue_directly+0x163/0x200 [94035.451734] blk_mq_request_issue_directly+0x4e/0xb0 [94035.451737] blk_mq_try_issue_list_directly+0x62/0x100 [94035.451741] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.451744] blk_mq_flush_plug_list+0x135/0x220 [94035.451748] blk_flush_plug_list+0xd7/0x100 [94035.451752] blk_finish_plug+0x25/0x36 [94035.451756] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.451788] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.451815] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.451840] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.451866] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.451891] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.451909] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.451926] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.452037] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.452148] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.452254] ? flush_work+0x42/0x1d0 [94035.452258] ? internal_add_timer+0x42/0x70 [94035.452260] ? _cond_resched+0x15/0x30 [94035.452262] ? mutex_lock+0xe/0x30 [94035.452265] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.452367] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.452468] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.452574] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.452680] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.452792] kthread+0x134/0x150 [94035.452795] ? set_kthread_struct+0x50/0x50 [94035.452798] ret_from_fork+0x1f/0x40 [94035.452803] CPU: 0 PID: 527429 Comm: ll_ost_io00_052 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.452808] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.452809] Call Trace: [94035.452812] dump_stack+0x41/0x60 [94035.452818] warn_alloc.cold.127+0x7b/0x108 [94035.452823] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.452827] ? __alloc_pages_nodemask+0x166/0x330 [94035.452830] __alloc_pages_nodemask+0x2e2/0x330 [94035.452834] kmalloc_order+0x28/0x90 [94035.452839] kmalloc_order_trace+0x1d/0xb0 [94035.452842] __kmalloc+0x203/0x250 [94035.452847] virtqueue_add+0x493/0xc70 [94035.452852] ? finish_wait+0x80/0x80 [94035.452856] virtqueue_add_sgs+0x80/0xa0 [94035.452860] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.452865] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.452870] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.452874] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.452878] scsi_queue_rq+0x512/0xb10 [94035.452880] __blk_mq_try_issue_directly+0x163/0x200 [94035.452885] blk_mq_request_issue_directly+0x4e/0xb0 [94035.452890] blk_mq_try_issue_list_directly+0x62/0x100 [94035.452893] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.452896] blk_mq_flush_plug_list+0x135/0x220 [94035.452901] blk_flush_plug_list+0xd7/0x100 [94035.452905] blk_finish_plug+0x25/0x36 [94035.452910] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.452944] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.452973] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.453002] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.453033] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.453064] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.453085] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.453104] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.456396] / [94035.456396] ,mems_allowed=0 [94035.458600] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.459529] ,mems_allowed=0 [94035.462018] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.465599] ? set_next_entity+0xb5/0x1e0 [94035.481952] ll_ost_io03_051: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [94035.482329] ? internal_add_timer+0x42/0x70 [94035.483630] ,cpuset= [94035.484726] ? _cond_resched+0x15/0x30 [94035.485720] / [94035.486961] ? mutex_lock+0xe/0x30 [94035.488156] ,mems_allowed=0 [94035.489447] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.491753] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.874830] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.875941] ? finish_wait+0x80/0x80 [94035.876629] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.877535] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.878470] kthread+0x134/0x150 [94035.879084] ? set_kthread_struct+0x50/0x50 [94035.879892] ret_from_fork+0x1f/0x40 [94035.880600] CPU: 12 PID: 527328 Comm: ll_ost_io03_051 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.883977] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.886236] Call Trace: [94035.887037] dump_stack+0x41/0x60 [94035.887991] warn_alloc.cold.127+0x7b/0x108 [94035.889172] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.890388] ? __alloc_pages_nodemask+0x166/0x330 [94035.891615] __alloc_pages_nodemask+0x2e2/0x330 [94035.892867] kmalloc_order+0x28/0x90 [94035.893803] kmalloc_order_trace+0x1d/0xb0 [94035.894910] __kmalloc+0x203/0x250 [94035.895925] virtqueue_add+0x493/0xc70 [94035.896887] ? finish_wait+0x80/0x80 [94035.897860] virtqueue_add_sgs+0x80/0xa0 [94035.898965] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.900347] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.901557] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.902872] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.904240] scsi_queue_rq+0x512/0xb10 [94035.905286] __blk_mq_try_issue_directly+0x163/0x200 [94035.906312] blk_mq_request_issue_directly+0x4e/0xb0 [94035.907234] blk_mq_try_issue_list_directly+0x62/0x100 [94035.908370] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.909306] blk_mq_flush_plug_list+0x135/0x220 [94035.910166] blk_flush_plug_list+0xd7/0x100 [94035.910960] blk_finish_plug+0x25/0x36 [94035.911688] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.912726] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.913678] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.914778] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.915900] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.916849] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.917722] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.918519] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.919445] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.920432] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.921479] ? flush_work+0x42/0x1d0 [94035.922204] ? internal_add_timer+0x42/0x70 [94035.922993] ? _cond_resched+0x15/0x30 [94035.923742] ? mutex_lock+0xe/0x30 [94035.924575] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.925779] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.926960] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.928143] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.929229] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.930545] kthread+0x134/0x150 [94035.931292] ? set_kthread_struct+0x50/0x50 [94035.932096] ret_from_fork+0x1f/0x40 [94035.932854] CPU: 4 PID: 527147 Comm: ll_ost_io01_043 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.935869] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.937847] Call Trace: [94035.938590] dump_stack+0x41/0x60 [94035.939448] warn_alloc.cold.127+0x7b/0x108 [94035.940539] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.941698] ? __alloc_pages_nodemask+0x166/0x330 [94035.942719] __alloc_pages_nodemask+0x2e2/0x330 [94035.943801] kmalloc_order+0x28/0x90 [94035.944742] kmalloc_order_trace+0x1d/0xb0 [94035.945649] __kmalloc+0x203/0x250 [94035.946439] virtqueue_add+0x493/0xc70 [94035.947330] ? finish_wait+0x80/0x80 [94035.948180] virtqueue_add_sgs+0x80/0xa0 [94035.948993] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.949998] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.950879] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94035.951904] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94035.953150] scsi_queue_rq+0x512/0xb10 [94035.953964] __blk_mq_try_issue_directly+0x163/0x200 [94035.954915] blk_mq_request_issue_directly+0x4e/0xb0 [94035.956052] blk_mq_try_issue_list_directly+0x62/0x100 [94035.957011] blk_mq_sched_insert_requests+0xa4/0xf0 [94035.957980] blk_mq_flush_plug_list+0x135/0x220 [94035.958882] blk_flush_plug_list+0xd7/0x100 [94035.959733] blk_finish_plug+0x25/0x36 [94035.960439] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94035.961485] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94035.962445] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.963557] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94035.964643] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94035.965597] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94035.966471] ofd_commitrw+0x5b4/0xd20 [ofd] [94035.967256] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.968258] obd_commitrw+0x1b6/0x370 [ptlrpc] [94035.969165] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94035.970107] ? internal_add_timer+0x42/0x70 [94035.970908] ? _cond_resched+0x15/0x30 [94035.971671] ? mutex_lock+0xe/0x30 [94035.972366] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94035.973381] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94035.974464] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94035.975580] ? finish_wait+0x80/0x80 [94035.976304] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94035.977262] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94035.978308] kthread+0x134/0x150 [94035.979044] ? set_kthread_struct+0x50/0x50 [94035.979847] ret_from_fork+0x1f/0x40 [94035.980541] CPU: 9 PID: 527247 Comm: ll_ost_io02_042 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94035.982992] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94035.985089] Call Trace: [94035.985846] dump_stack+0x41/0x60 [94035.986746] warn_alloc.cold.127+0x7b/0x108 [94035.987839] __alloc_pages_slowpath+0xcb2/0xcd0 [94035.988996] ? __alloc_pages_nodemask+0x166/0x330 [94035.989987] __alloc_pages_nodemask+0x2e2/0x330 [94035.991072] kmalloc_order+0x28/0x90 [94035.992022] kmalloc_order_trace+0x1d/0xb0 [94035.992945] __kmalloc+0x203/0x250 [94035.993900] virtqueue_add+0x493/0xc70 [94035.994918] ? finish_wait+0x80/0x80 [94035.995820] virtqueue_add_sgs+0x80/0xa0 [94035.996870] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94035.997890] ? scsi_alloc_sgtables+0x84/0x1c0 [94035.998819] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94036.000057] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94036.001465] scsi_queue_rq+0x512/0xb10 [94036.002506] __blk_mq_try_issue_directly+0x163/0x200 [94036.003765] blk_mq_request_issue_directly+0x4e/0xb0 [94036.004918] blk_mq_try_issue_list_directly+0x62/0x100 [94036.006239] blk_mq_sched_insert_requests+0xa4/0xf0 [94036.007377] blk_mq_flush_plug_list+0x135/0x220 [94036.008518] blk_flush_plug_list+0xd7/0x100 [94036.009603] blk_finish_plug+0x25/0x36 [94036.010630] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94036.012139] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94036.013389] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94036.014846] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94036.016353] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94036.017672] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94036.018643] ofd_commitrw+0x5b4/0xd20 [ofd] [94036.019726] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94036.021029] obd_commitrw+0x1b6/0x370 [ptlrpc] [94036.022305] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94036.023446] ? flush_work+0x42/0x1d0 [94036.024331] ? internal_add_timer+0x42/0x70 [94036.025409] ? _cond_resched+0x15/0x30 [94036.026413] ? mutex_lock+0xe/0x30 [94036.027335] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94036.028748] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94036.030245] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94036.031774] ? finish_wait+0x80/0x80 [94036.032758] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94036.034030] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94036.035432] kthread+0x134/0x150 [94036.036340] ? set_kthread_struct+0x50/0x50 [94036.037488] ret_from_fork+0x1f/0x40 [94058.532782] warn_alloc: 1 callbacks suppressed [94058.532785] ll_ost_io02_002: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94058.537097] CPU: 9 PID: 188105 Comm: ll_ost_io02_002 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94058.540103] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94058.541680] ll_ost_io02_017: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [94058.542353] Call Trace: [94058.542360] dump_stack+0x41/0x60 [94058.544988] / [94058.545916] warn_alloc.cold.127+0x7b/0x108 [94058.546988] ,mems_allowed=0 [94058.547705] __alloc_pages_slowpath+0xcb2/0xcd0 [94058.549920] __alloc_pages_nodemask+0x2e2/0x330 [94058.553344] kmalloc_order+0x28/0x90 [94058.554431] kmalloc_order_trace+0x1d/0xb0 [94058.555630] __kmalloc+0x203/0x250 [94058.556686] virtqueue_add+0x493/0xc70 [94058.557783] ? finish_wait+0x80/0x80 [94058.558858] virtqueue_add_sgs+0x80/0xa0 [94058.559967] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94058.561446] ? scsi_alloc_sgtables+0x84/0x1c0 [94058.562723] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94058.563550] ll_ost_io03_051: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94058.564147] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94058.568056] scsi_queue_rq+0x512/0xb10 [94058.569179] __blk_mq_try_issue_directly+0x163/0x200 [94058.570535] blk_mq_request_issue_directly+0x4e/0xb0 [94058.571901] blk_mq_try_issue_list_directly+0x62/0x100 [94058.573293] blk_mq_sched_insert_requests+0xa4/0xf0 [94058.574610] blk_mq_flush_plug_list+0x135/0x220 [94058.575825] blk_flush_plug_list+0xd7/0x100 [94058.576970] blk_finish_plug+0x25/0x36 [94058.578058] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94058.579562] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94058.580899] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94058.582261] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94058.583829] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94058.585093] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94058.586351] ofd_commitrw+0x5b4/0xd20 [ofd] [94058.587425] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94058.588728] obd_commitrw+0x1b6/0x370 [ptlrpc] [94058.590017] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94058.591346] ? flush_work+0x42/0x1d0 [94058.592349] ? internal_add_timer+0x42/0x70 [94058.593422] ? _cond_resched+0x15/0x30 [94058.594470] ? mutex_lock+0xe/0x30 [94058.595434] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94058.596853] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94058.598426] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94058.600038] ? finish_wait+0x80/0x80 [94058.601047] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94058.602321] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94058.603692] kthread+0x134/0x150 [94058.604608] ? set_kthread_struct+0x50/0x50 [94058.605684] ret_from_fork+0x1f/0x40 [94058.606634] warn_alloc_show_mem: 9 callbacks suppressed [94058.606637] Mem-Info: [94058.606635] CPU: 10 PID: 527169 Comm: ll_ost_io02_017 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94058.607886] active_anon:27724 inactive_anon:307431 isolated_anon:0 active_file:31088330 inactive_file:3342482 isolated_file:31 unevictable:0 dirty:803 writeback:0 slab_reclaimable:849840 slab_unreclaimable:621955 mapped:20675 shmem:148 pagetables:5585 bounce:0 free:1775005 free_pcp:463 free_cma:0 [94058.608491] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94058.611362] Node 0 active_anon:110896kB inactive_anon:1229724kB active_file:124353320kB inactive_file:13369928kB unevictable:0kB isolated(anon):0kB isolated(file):124kB mapped:82700kB dirty:3212kB writeback:0kB shmem:592kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35296kB pagetables:22340kB all_unreclaimable? no [94058.617117] Call Trace: [94058.619138] Node 0 [94058.624345] dump_stack+0x41/0x60 [94058.625174] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94058.625735] warn_alloc.cold.127+0x7b/0x108 [94058.626705] lowmem_reserve[]: [94058.630335] __alloc_pages_slowpath+0xcb2/0xcd0 [94058.631523] 0 [94058.632211] __alloc_pages_nodemask+0x2e2/0x330 [94058.633468] 913 [94058.633981] kmalloc_order+0x28/0x90 [94058.635209] 150007 [94058.635721] kmalloc_order_trace+0x1d/0xb0 [94058.636754] 150007 [94058.637309] __kmalloc+0x203/0x250 [94058.638448] 150007 [94058.639015] virtqueue_add+0x493/0xc70 [94058.640585] ? finish_wait+0x80/0x80 [94058.641704] Node 0 [94058.642202] virtqueue_add_sgs+0x80/0xa0 [94058.643237] DMA32 free:596548kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:1488kB active_file:252kB inactive_file:252kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94058.643776] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94058.644859] lowmem_reserve[]: [94058.648660] ? scsi_alloc_sgtables+0x84/0x1c0 [94058.650059] 0 [94058.650735] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94058.651943] 0 [94058.652459] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94058.653785] 149093 [94058.654290] scsi_queue_rq+0x512/0xb10 [94058.655733] 149093 [94058.656308] __blk_mq_try_issue_directly+0x163/0x200 [94058.657392] 149093 [94058.657954] blk_mq_request_issue_directly+0x4e/0xb0 [94058.659856] blk_mq_try_issue_list_directly+0x62/0x100 [94058.661181] Node 0 [94058.661684] blk_mq_sched_insert_requests+0xa4/0xf0 [94058.663034] Normal free:6493048kB min:51316kB low:203988kB high:356660kB active_anon:110516kB inactive_anon:1228236kB active_file:124331164kB inactive_file:13390192kB unevictable:0kB writepending:2100kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1892kB local_pcp:16kB free_cma:0kB [94058.663606] blk_mq_flush_plug_list+0x135/0x220 [94058.664897] lowmem_reserve[]: [94058.669149] blk_flush_plug_list+0xd7/0x100 [94058.670380] 0 [94058.671085] blk_finish_plug+0x25/0x36 [94058.672284] 0 [94058.672808] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94058.673891] 0 [94058.674391] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94058.675859] 0 [94058.676361] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94058.677711] 0 [94058.678338] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94058.680277] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94058.681809] Node 0 [94058.682285] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94058.683634] DMA: [94058.684189] ofd_commitrw+0x5b4/0xd20 [ofd] [94058.685448] 0*4kB [94058.685967] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94058.687097] 0*8kB [94058.687650] obd_commitrw+0x1b6/0x370 [ptlrpc] [94058.688885] 0*16kB [94058.689429] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94058.690601] 0*32kB [94058.691253] ? flush_work+0x42/0x1d0 [94058.692545] 0*64kB [94058.693264] ? internal_add_timer+0x42/0x70 [94058.694294] 0*128kB [94058.694917] ? _cond_resched+0x15/0x30 [94058.696084] 0*256kB [94058.696655] ? mutex_lock+0xe/0x30 [94058.697714] 0*512kB [94058.698281] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94058.699252] 1*1024kB [94058.699819] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94058.701107] (U) [94058.701689] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94058.703105] 1*2048kB [94058.703625] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94058.705032] (M) [94058.705591] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94058.706737] 2*4096kB [94058.707247] kthread+0x134/0x150 [94058.708545] (M) [94058.709118] ? set_kthread_struct+0x50/0x50 [94058.710069] = 11264kB [94058.710570] ret_from_fork+0x1f/0x40 [94058.711700] Node 0 [94058.712266] CPU: 14 PID: 527328 Comm: ll_ost_io03_051 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94058.713378] DMA32: [94058.714096] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94058.718171] 119*4kB [94058.718975] Call Trace: [94058.721057] (UMH) [94058.721883] dump_stack+0x41/0x60 [94058.722707] 111*8kB [94058.723450] warn_alloc.cold.127+0x7b/0x108 [94058.724425] (UMH) [94058.725184] __alloc_pages_slowpath+0xcb2/0xcd0 [94058.726297] 101*16kB [94058.727031] __alloc_pages_nodemask+0x2e2/0x330 [94058.728229] (UMEH) [94058.728995] kmalloc_order+0x28/0x90 [94058.730184] 85*32kB [94058.730906] kmalloc_order_trace+0x1d/0xb0 [94058.731927] (UMH) [94058.732667] __kmalloc+0x203/0x250 [94058.733763] 108*64kB [94058.734472] ? __wake_up_common+0x7a/0x190 [94058.735435] (UMEH) [94058.736217] virtqueue_add+0x493/0xc70 [94058.737338] 120*128kB [94058.738088] ? finish_wait+0x80/0x80 [94058.739122] (UMH) [94058.739929] virtqueue_add_sgs+0x80/0xa0 [94058.740954] 106*256kB [94058.741705] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94058.742781] (UMEH) [94058.743586] ? scsi_alloc_sgtables+0x84/0x1c0 [94058.744979] 98*512kB [94058.745767] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94058.746934] (UM) [94058.747734] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94058.749018] 78*1024kB [94058.749769] scsi_queue_rq+0x512/0xb10 [94058.751174] (UMEH) [94058.751950] __blk_mq_try_issue_directly+0x163/0x200 [94058.752998] 57*2048kB [94058.753744] blk_mq_request_issue_directly+0x4e/0xb0 [94058.755026] (UME) [94058.755846] blk_mq_try_issue_list_directly+0x62/0x100 [94058.757131] 72*4096kB [94058.757884] blk_mq_sched_insert_requests+0xa4/0xf0 [94058.759185] (UM) [94058.759998] blk_mq_flush_plug_list+0x135/0x220 [94058.761264] = 596804kB [94058.761998] blk_flush_plug_list+0xd7/0x100 [94058.763164] Node 0 [94058.763975] blk_finish_plug+0x25/0x36 [94058.765099] Normal: [94058.765864] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94058.766887] 68344*4kB [94058.767659] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94058.769054] (UME) [94058.769847] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94058.771128] 676402*8kB [94058.771847] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94058.773332] (UME) [94058.774130] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94058.775567] 50359*16kB [94058.776283] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94058.777581] (UMH) [94058.778390] ofd_commitrw+0x5b4/0xd20 [ofd] [94058.779598] 14*32kB [94058.780315] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94058.781374] (UH) [94058.782147] obd_commitrw+0x1b6/0x370 [ptlrpc] [94058.783299] 3*64kB [94058.784009] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94058.785123] (H) [94058.785906] ? newidle_balance+0x2b6/0x3b0 [94058.787093] 0*128kB [94058.787813] ? internal_add_timer+0x42/0x70 [94058.788856] 0*256kB [94058.789626] ? _cond_resched+0x15/0x30 [94058.790731] 0*512kB [94058.791520] ? mutex_lock+0xe/0x30 [94058.792547] 0*1024kB [94058.793306] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94058.794235] 0*2048kB [94058.795009] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94058.796246] 0*4096kB [94058.797003] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94058.798371] = 6490976kB [94058.799147] ? finish_wait+0x80/0x80 [94058.800591] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [94058.801406] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94058.802396] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [94058.804359] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94058.805534] 34429847 total pagecache pages [94058.807489] kthread+0x134/0x150 [94058.808780] 1633 pages in swap cache [94058.809913] ? set_kthread_struct+0x50/0x50 [94058.810898] Swap cache stats: add 1413209, delete 1411590, find 67494/81265 [94058.811945] ret_from_fork+0x1f/0x40 [94058.813087] Free swap = 5976612kB [94058.816681] Total swap = 11075580kB [94058.817613] 39321462 pages RAM [94058.818477] 0 pages HighMem/MovableOnly [94058.819465] 905848 pages reserved [94058.820370] 0 pages hwpoisoned [94059.619176] ll_ost_io02_073: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94059.621413] ll_ost_io04_038: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [94059.622202] CPU: 9 PID: 527864 Comm: ll_ost_io02_073 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94059.622740] ll_ost_io01_054: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [94059.623277] ll_ost_io00_078: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [94059.625007] / [94059.625006] ,mems_allowed=0 [94059.627938] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94059.630546] / [94059.630545] ,mems_allowed=0 [94059.633106] Call Trace: [94059.633637] ,mems_allowed=0 [94059.634559] dump_stack+0x41/0x60 [94059.636781] warn_alloc.cold.127+0x7b/0x108 [94059.641246] __alloc_pages_slowpath+0xcb2/0xcd0 [94059.642220] ? __alloc_pages_nodemask+0x166/0x330 [94059.643398] __alloc_pages_nodemask+0x2e2/0x330 [94059.644395] kmalloc_order+0x28/0x90 [94059.645381] kmalloc_order_trace+0x1d/0xb0 [94059.646265] __kmalloc+0x203/0x250 [94059.647207] virtqueue_add+0x493/0xc70 [94059.648248] ? finish_wait+0x80/0x80 [94059.648996] virtqueue_add_sgs+0x80/0xa0 [94059.649764] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94059.650739] ? scsi_alloc_sgtables+0x84/0x1c0 [94059.651556] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94059.652466] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94059.653530] scsi_queue_rq+0x512/0xb10 [94059.654320] __blk_mq_try_issue_directly+0x163/0x200 [94059.655227] blk_mq_request_issue_directly+0x4e/0xb0 [94059.656125] blk_mq_try_issue_list_directly+0x62/0x100 [94059.657046] blk_mq_sched_insert_requests+0xa4/0xf0 [94059.657921] blk_mq_flush_plug_list+0x135/0x220 [94059.658756] blk_flush_plug_list+0xd7/0x100 [94059.659551] blk_finish_plug+0x25/0x36 [94059.660259] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94059.661288] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [94059.662239] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.663349] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.664440] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94059.665397] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94059.666276] ofd_commitrw+0x5b4/0xd20 [ofd] [94059.667101] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.668080] obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.669006] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94059.670001] ? flush_work+0x42/0x1d0 [94059.670739] ? internal_add_timer+0x42/0x70 [94059.671529] ? _cond_resched+0x15/0x30 [94059.672241] ? mutex_lock+0xe/0x30 [94059.672893] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94059.673861] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94059.674919] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94059.675998] ? tgt_key_exit+0xc/0x20 [ptlrpc] [94059.676869] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94059.677745] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94059.678705] kthread+0x134/0x150 [94059.679321] ? set_kthread_struct+0x50/0x50 [94059.680098] ret_from_fork+0x1f/0x40 [94059.680774] warn_alloc_show_mem: 2 callbacks suppressed [94059.680776] Mem-Info: [94059.680775] CPU: 3 PID: 527783 Comm: ll_ost_io00_078 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94059.681679] active_anon:27724 inactive_anon:309259 isolated_anon:0 active_file:31221239 inactive_file:3320007 isolated_file:31 unevictable:0 dirty:1004 writeback:0 slab_reclaimable:850534 slab_unreclaimable:621933 mapped:21362 shmem:148 pagetables:5588 bounce:0 free:1661217 free_pcp:441 free_cma:0 [94059.682243] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94059.684319] Node 0 active_anon:110896kB inactive_anon:1237036kB active_file:124884956kB inactive_file:13280028kB unevictable:0kB isolated(anon):0kB isolated(file):124kB mapped:85448kB dirty:4016kB writeback:0kB shmem:592kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35344kB pagetables:22352kB all_unreclaimable? no [94059.690114] Call Trace: [94059.691666] Node 0 [94059.696630] dump_stack+0x41/0x60 [94059.697232] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94059.697769] warn_alloc.cold.127+0x7b/0x108 [94059.698476] lowmem_reserve[]: [94059.702093] __alloc_pages_slowpath+0xcb2/0xcd0 [94059.702922] 0 [94059.703580] ? __alloc_pages_nodemask+0x166/0x330 [94059.704462] 913 [94059.704964] __alloc_pages_nodemask+0x2e2/0x330 [94059.705881] 150007 [94059.706404] kmalloc_order+0x28/0x90 [94059.707298] 150007 [94059.707847] kmalloc_order_trace+0x1d/0xb0 [94059.708592] 150007 [94059.709141] __kmalloc+0x203/0x250 [94059.710513] virtqueue_add+0x493/0xc70 [94059.711243] Node 0 [94059.711697] ? finish_wait+0x80/0x80 [94059.712474] DMA32 free:596708kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:1488kB active_file:252kB inactive_file:252kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94059.713023] virtqueue_add_sgs+0x80/0xa0 [94059.713767] lowmem_reserve[]: [94059.717628] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94059.718458] 0 [94059.719143] ? scsi_alloc_sgtables+0x84/0x1c0 [94059.720178] 0 [94059.720671] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94059.721549] 149093 [94059.722048] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94059.723046] 149093 [94059.723591] scsi_queue_rq+0x512/0xb10 [94059.724665] 149093 [94059.725219] __blk_mq_try_issue_directly+0x163/0x200 [94059.726570] blk_mq_request_issue_directly+0x4e/0xb0 [94059.727757] Node 0 [94059.728238] blk_mq_try_issue_list_directly+0x62/0x100 [94059.729211] Normal free:6028816kB min:49268kB low:201940kB high:354612kB active_anon:110516kB inactive_anon:1235324kB active_file:124866856kB inactive_file:13307004kB unevictable:0kB writepending:4016kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2060kB local_pcp:108kB free_cma:0kB [94059.729759] blk_mq_sched_insert_requests+0xa4/0xf0 [94059.730738] lowmem_reserve[]: [94059.735057] blk_mq_flush_plug_list+0x135/0x220 [94059.736038] 0 [94059.736712] blk_flush_plug_list+0xd7/0x100 [94059.737626] 0 [94059.738144] blk_finish_plug+0x25/0x36 [94059.739005] 0 [94059.739492] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94059.740279] 0 [94059.740772] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94059.741841] 0 [94059.742341] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.743814] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.744949] Node 0 [94059.745428] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94059.746551] DMA: [94059.747114] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94059.748115] 0*4kB [94059.748632] ofd_commitrw+0x5b4/0xd20 [ofd] [94059.749564] 0*8kB [94059.750109] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.750927] 0*16kB [94059.751480] obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.752363] 0*32kB [94059.752926] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94059.753960] 0*64kB [94059.754513] ? flush_work+0x42/0x1d0 [94059.755468] 0*128kB [94059.756042] ? internal_add_timer+0x42/0x70 [94059.756806] 0*256kB [94059.757378] ? _cond_resched+0x15/0x30 [94059.758226] 0*512kB [94059.758785] ? mutex_lock+0xe/0x30 [94059.759709] 1*1024kB [94059.760255] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94059.760978] (U) [94059.761552] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94059.762504] 1*2048kB [94059.762993] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94059.764198] (M) [94059.764778] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94059.765888] 2*4096kB [94059.766416] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94059.767291] (M) [94059.767884] kthread+0x134/0x150 [94059.768876] = 11264kB [94059.769414] ? set_kthread_struct+0x50/0x50 [94059.770142] Node 0 [94059.770720] ret_from_fork+0x1f/0x40 [94059.771576] DMA32: [94059.772120] CPU: 17 PID: 527566 Comm: ll_ost_io04_038 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94059.772847] 119*4kB [94059.773751] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94059.776788] (UMH) [94059.777617] Call Trace: [94059.779439] 111*8kB [94059.780202] dump_stack+0x41/0x60 [94059.781043] (UMH) [94059.781829] warn_alloc.cold.127+0x7b/0x108 [94059.782833] 101*16kB [94059.783582] __alloc_pages_slowpath+0xcb2/0xcd0 [94059.784744] (UMEH) [94059.785548] ? __alloc_pages_nodemask+0x166/0x330 [94059.786793] 94*32kB [94059.787548] __alloc_pages_nodemask+0x2e2/0x330 [94059.788850] (UMH) [94059.789429] kmalloc_order+0x28/0x90 [94059.790488] 112*64kB [94059.791139] kmalloc_order_trace+0x1d/0xb0 [94059.791925] (UMEH) [94059.792703] __kmalloc+0x203/0x250 [94059.793580] 119*128kB [94059.794258] virtqueue_add+0x493/0xc70 [94059.795269] (UM) [94059.796068] ? finish_wait+0x80/0x80 [94059.797150] 107*256kB [94059.797702] virtqueue_add_sgs+0x80/0xa0 [94059.798731] (UMEH) [94059.799378] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94059.800340] 98*512kB [94059.800922] ? scsi_alloc_sgtables+0x84/0x1c0 [94059.801976] (UM) [94059.802766] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94059.803674] 78*1024kB [94059.804233] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94059.805271] (UMEH) [94059.806045] scsi_queue_rq+0x512/0xb10 [94059.807153] 57*2048kB [94059.807912] __blk_mq_try_issue_directly+0x163/0x200 [94059.808753] (UME) [94059.809518] blk_mq_request_issue_directly+0x4e/0xb0 [94059.810634] 72*4096kB [94059.811332] blk_mq_try_issue_list_directly+0x62/0x100 [94059.812635] (UM) [94059.813365] blk_mq_sched_insert_requests+0xa4/0xf0 [94059.814381] = 597476kB [94059.815106] blk_mq_flush_plug_list+0x135/0x220 [94059.816444] Node 0 [94059.817266] blk_flush_plug_list+0xd7/0x100 [94059.818520] Normal: [94059.819286] blk_finish_plug+0x25/0x36 [94059.820459] 15409*4kB [94059.821256] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94059.822253] (UME) [94059.822913] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94059.824388] 643529*8kB [94059.825110] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.826451] (UME) [94059.827225] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.828456] 50292*16kB [94059.829163] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94059.830321] (UMH) [94059.831106] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94059.832169] 20*32kB [94059.832871] ofd_commitrw+0x5b4/0xd20 [ofd] [94059.833841] (UMEH) [94059.834465] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.835299] 0*64kB [94059.835849] obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.836777] 0*128kB [94059.837484] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94059.838624] 0*256kB [94059.839196] ? set_next_entity+0xb5/0x1e0 [94059.840432] 0*512kB [94059.841194] ? internal_add_timer+0x42/0x70 [94059.842330] 0*1024kB [94059.842902] ? _cond_resched+0x15/0x30 [94059.844071] 0*2048kB [94059.844640] ? mutex_lock+0xe/0x30 [94059.845698] 0*4096kB [94059.846283] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94059.847250] = 6015180kB [94059.847844] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94059.849147] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [94059.849871] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94059.851328] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [94059.852830] ? finish_wait+0x80/0x80 [94059.854304] 34547798 total pagecache pages [94059.855765] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94059.856755] 1644 pages in swap cache [94059.857613] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94059.858771] Swap cache stats: add 1413286, delete 1411660, find 67494/81265 [94059.859514] kthread+0x134/0x150 [94059.860830] Free swap = 5976356kB [94059.862119] ? set_kthread_struct+0x50/0x50 [94059.863064] Total swap = 11075580kB [94059.863794] ret_from_fork+0x1f/0x40 [94059.864930] 39321462 pages RAM [94059.865699] CPU: 6 PID: 527158 Comm: ll_ost_io01_054 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94059.866661] 0 pages HighMem/MovableOnly [94059.867693] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94059.870668] 905848 pages reserved [94059.871814] Call Trace: [94059.873972] 0 pages hwpoisoned [94059.875007] dump_stack+0x41/0x60 [94059.877788] warn_alloc.cold.127+0x7b/0x108 [94059.878962] __alloc_pages_slowpath+0xcb2/0xcd0 [94059.880214] ? __alloc_pages_nodemask+0x166/0x330 [94059.881557] __alloc_pages_nodemask+0x2e2/0x330 [94059.882814] kmalloc_order+0x28/0x90 [94059.883867] kmalloc_order_trace+0x1d/0xb0 [94059.885006] __kmalloc+0x203/0x250 [94059.885978] ? __wake_up_common+0x7a/0x190 [94059.887101] virtqueue_add+0x493/0xc70 [94059.888162] ? finish_wait+0x80/0x80 [94059.889185] virtqueue_add_sgs+0x80/0xa0 [94059.890290] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94059.891730] ? scsi_alloc_sgtables+0x84/0x1c0 [94059.892930] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94059.894279] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94059.895757] scsi_queue_rq+0x512/0xb10 [94059.896841] __blk_mq_try_issue_directly+0x163/0x200 [94059.898209] blk_mq_request_issue_directly+0x4e/0xb0 [94059.899544] blk_mq_try_issue_list_directly+0x62/0x100 [94059.900943] blk_mq_sched_insert_requests+0xa4/0xf0 [94059.902279] blk_mq_flush_plug_list+0x135/0x220 [94059.903564] blk_flush_plug_list+0xd7/0x100 [94059.904770] blk_finish_plug+0x25/0x36 [94059.905879] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94059.907430] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94059.908850] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.910508] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94059.912110] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94059.913522] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94059.914838] ofd_commitrw+0x5b4/0xd20 [ofd] [94059.916017] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.917383] obd_commitrw+0x1b6/0x370 [ptlrpc] [94059.918716] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94059.920102] ? flush_work+0x42/0x1d0 [94059.921145] ? internal_add_timer+0x42/0x70 [94059.922325] ? _cond_resched+0x15/0x30 [94059.923390] ? mutex_lock+0xe/0x30 [94059.924346] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94059.925783] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94059.927342] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94059.928925] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94059.930226] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94059.931676] kthread+0x134/0x150 [94059.932611] ? set_kthread_struct+0x50/0x50 [94059.933717] ret_from_fork+0x1f/0x40 [94962.833692] ll_ost_io02_066: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94962.837242] CPU: 10 PID: 527703 Comm: ll_ost_io02_066 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94962.840295] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94962.842544] Call Trace: [94962.843473] dump_stack+0x41/0x60 [94962.844504] warn_alloc.cold.127+0x7b/0x108 [94962.845702] __alloc_pages_slowpath+0xcb2/0xcd0 [94962.847010] ? __alloc_pages_nodemask+0x166/0x330 [94962.848323] __alloc_pages_nodemask+0x2e2/0x330 [94962.849598] kmalloc_order+0x28/0x90 [94962.850747] kmalloc_order_trace+0x1d/0xb0 [94962.851935] __kmalloc+0x203/0x250 [94962.852937] virtqueue_add+0x493/0xc70 [94962.854034] virtqueue_add_sgs+0x80/0xa0 [94962.855138] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94962.856577] ? scsi_alloc_sgtables+0x84/0x1c0 [94962.857753] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94962.859059] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94962.860491] scsi_queue_rq+0x512/0xb10 [94962.861553] __blk_mq_try_issue_directly+0x163/0x200 [94962.862868] blk_mq_request_issue_directly+0x4e/0xb0 [94962.864155] blk_mq_try_issue_list_directly+0x62/0x100 [94962.865444] blk_mq_sched_insert_requests+0xa4/0xf0 [94962.866685] blk_mq_flush_plug_list+0x135/0x220 [94962.867881] blk_flush_plug_list+0xd7/0x100 [94962.869041] blk_finish_plug+0x25/0x36 [94962.870081] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94962.871676] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94962.873211] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94962.875275] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94962.876842] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94962.878193] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94962.879470] ofd_commitrw+0x5b4/0xd20 [ofd] [94962.880573] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94962.881925] obd_commitrw+0x1b6/0x370 [ptlrpc] [94962.883293] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94962.884694] ? flush_work+0x42/0x1d0 [94962.885684] ? internal_add_timer+0x42/0x70 [94962.886773] ? _cond_resched+0x15/0x30 [94962.887779] ? mutex_lock+0xe/0x30 [94962.888736] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94962.890129] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94962.891567] ll_ost_io01_014: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94962.891646] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94962.892193] ll_ost_io04_047: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [94962.898822] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94962.900105] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94962.901505] kthread+0x134/0x150 [94962.902422] ? set_kthread_struct+0x50/0x50 [94962.903551] ret_from_fork+0x1f/0x40 [94962.904572] warn_alloc_show_mem: 3 callbacks suppressed [94962.904575] Mem-Info: [94962.904573] CPU: 19 PID: 527684 Comm: ll_ost_io04_047 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94962.905900] active_anon:27759 inactive_anon:262732 isolated_anon:0 active_file:28557352 inactive_file:6840955 isolated_file:32 unevictable:0 dirty:1289 writeback:2 slab_reclaimable:909295 slab_unreclaimable:605377 mapped:22432 shmem:146 pagetables:5617 bounce:0 free:812320 free_pcp:512 free_cma:0 [94962.906744] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94962.909601] Node 0 active_anon:111036kB inactive_anon:1050928kB active_file:114229408kB inactive_file:27363820kB unevictable:0kB isolated(anon):0kB isolated(file):128kB mapped:89728kB dirty:5156kB writeback:8kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35344kB pagetables:22468kB all_unreclaimable? no [94962.917705] Call Trace: [94962.919952] Node 0 [94962.926808] dump_stack+0x41/0x60 [94962.927679] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94962.928486] warn_alloc.cold.127+0x7b/0x108 [94962.929532] lowmem_reserve[]: [94962.934697] __alloc_pages_slowpath+0xcb2/0xcd0 [94962.935925] 0 [94962.936913] ? __alloc_pages_nodemask+0x166/0x330 [94962.938231] 913 [94962.938968] __alloc_pages_nodemask+0x2e2/0x330 [94962.940314] 150007 [94962.941072] kmalloc_order+0x28/0x90 [94962.942384] 150007 [94962.943202] kmalloc_order_trace+0x1d/0xb0 [94962.944316] 150007 [94962.945140] __kmalloc+0x203/0x250 [94962.947206] virtqueue_add+0x493/0xc70 [94962.948301] Node 0 [94962.949027] virtqueue_add_sgs+0x80/0xa0 [94962.950184] DMA32 free:596580kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:424kB active_file:340kB inactive_file:56kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [94962.951003] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94962.952191] lowmem_reserve[]: 0 [94962.957754] ? scsi_alloc_sgtables+0x84/0x1c0 [94962.959246] 0 [94962.960272] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94962.961570] 149093 [94962.962332] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94962.963763] 149093 [94962.964580] scsi_queue_rq+0x512/0xb10 [94962.966099] 149093 [94962.966937] __blk_mq_try_issue_directly+0x163/0x200 [94962.968921] blk_mq_request_issue_directly+0x4e/0xb0 [94962.970292] Node 0 [94962.971014] blk_mq_try_issue_list_directly+0x62/0x100 [94962.972364] Normal free:2607128kB min:49268kB low:201940kB high:354612kB active_anon:110656kB inactive_anon:1050504kB active_file:114268304kB inactive_file:27357820kB unevictable:0kB writepending:5164kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2284kB local_pcp:108kB free_cma:0kB [94962.973191] blk_mq_sched_insert_requests+0xa4/0xf0 [94962.974618] lowmem_reserve[]: [94962.980588] blk_mq_flush_plug_list+0x135/0x220 [94962.981940] 0 [94962.982924] blk_flush_plug_list+0xd7/0x100 [94962.984201] 0 [94962.984906] blk_finish_plug+0x25/0x36 [94962.986105] 0 [94962.986817] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94962.987933] 0 [94962.988625] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94962.990108] 0 [94962.990782] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94962.992810] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94962.994419] Node 0 [94962.995056] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94962.996621] DMA: [94962.997380] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94962.998777] 0*4kB [94962.999436] ofd_commitrw+0x5b4/0xd20 [ofd] [94963.000727] 0*8kB [94963.001454] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94963.002591] 0*16kB [94963.003340] obd_commitrw+0x1b6/0x370 [ptlrpc] [94963.004566] 0*32kB [94963.005296] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94963.006432] 0*64kB [94963.007206] ? flush_work+0x42/0x1d0 [94963.008470] 0*128kB [94963.009250] ? internal_add_timer+0x42/0x70 [94963.010333] 0*256kB [94963.011127] ? _cond_resched+0x15/0x30 [94963.012306] 0*512kB [94963.013092] ? mutex_lock+0xe/0x30 [94963.014190] 1*1024kB [94963.014961] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94963.015934] (U) [94963.016718] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94963.018019] 1*2048kB [94963.018732] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94963.020171] (M) [94963.020812] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94963.022264] 2*4096kB [94963.022957] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94963.024126] (M) [94963.024910] kthread+0x134/0x150 [94963.026242] = 11264kB [94963.026955] ? set_kthread_struct+0x50/0x50 [94963.027948] Node 0 [94963.028707] ret_from_fork+0x1f/0x40 [94963.029945] DMA32: [94963.030631] CPU: 6 PID: 527067 Comm: ll_ost_io01_014 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [94963.031733] 47*4kB [94963.032511] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [94963.035664] (UMH) [94963.036550] Call Trace: [94963.038917] 69*8kB [94963.039704] dump_stack+0x41/0x60 [94963.040608] (UMH) [94963.041373] warn_alloc.cold.127+0x7b/0x108 [94963.042388] 76*16kB [94963.043112] __alloc_pages_slowpath+0xcb2/0xcd0 [94963.044331] (UMEH) [94963.045147] __alloc_pages_nodemask+0x2e2/0x330 [94963.046438] 60*32kB [94963.047194] kmalloc_order+0x28/0x90 [94963.048473] (UMH) [94963.049251] kmalloc_order_trace+0x1d/0xb0 [94963.050276] 94*64kB [94963.051010] __kmalloc+0x203/0x250 [94963.052138] (UMH) [94963.052859] virtqueue_add+0x493/0xc70 [94963.053845] 97*128kB [94963.054568] virtqueue_add_sgs+0x80/0xa0 [94963.055641] (UMEH) [94963.056445] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [94963.057527] 88*256kB [94963.058265] ? scsi_alloc_sgtables+0x84/0x1c0 [94963.059733] (UM) [94963.060466] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [94963.061658] 86*512kB [94963.062400] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [94963.063730] (UMH) [94963.064520] scsi_queue_rq+0x512/0xb10 [94963.065964] 72*1024kB [94963.066681] __blk_mq_try_issue_directly+0x163/0x200 [94963.067749] (UMEH) [94963.068552] blk_mq_request_issue_directly+0x4e/0xb0 [94963.069854] 54*2048kB [94963.070588] blk_mq_try_issue_list_directly+0x62/0x100 [94963.071884] (UM) [94963.072704] blk_mq_sched_insert_requests+0xa4/0xf0 [94963.074061] 79*4096kB [94963.074786] blk_mq_flush_plug_list+0x135/0x220 [94963.076108] (UM) [94963.076904] blk_flush_plug_list+0xd7/0x100 [94963.078132] = 596772kB [94963.078845] blk_finish_plug+0x25/0x36 [94963.079996] Node 0 [94963.080782] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [94963.081827] Normal: [94963.082559] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [94963.084022] 415685*4kB [94963.084776] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94963.086085] (UME) [94963.086888] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [94963.088423] 96519*8kB [94963.089137] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [94963.090628] (UME) [94963.091405] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [94963.092716] 6332*16kB [94963.093428] ofd_commitrw+0x5b4/0xd20 [ofd] [94963.094669] (UMEH) [94963.095421] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [94963.096516] 19*32kB [94963.097241] obd_commitrw+0x1b6/0x370 [ptlrpc] [94963.098454] (UH) [94963.099201] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [94963.100367] 3*64kB [94963.101049] ? flush_work+0x42/0x1d0 [94963.102312] (H) [94963.103071] ? internal_add_timer+0x42/0x70 [94963.104103] 0*128kB [94963.104792] ? _cond_resched+0x15/0x30 [94963.105959] 0*256kB [94963.106730] ? mutex_lock+0xe/0x30 [94963.107808] 0*512kB [94963.108551] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [94963.109486] 0*1024kB [94963.110285] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [94963.111555] 0*2048kB [94963.112314] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [94963.113739] 0*4096kB [94963.114513] ptlrpc_main+0xbec/0x1530 [ptlrpc] [94963.115942] = 2537004kB [94963.116702] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [94963.117866] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [94963.118667] kthread+0x134/0x150 [94963.119982] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [94963.122034] ? set_kthread_struct+0x50/0x50 [94963.123008] 35432331 total pagecache pages [94963.124964] ret_from_fork+0x1f/0x40 [94963.126137] 1385 pages in swap cache [94963.129367] Swap cache stats: add 1481705, delete 1480323, find 71953/86670 [94963.131025] Free swap = 5739668kB [94963.132062] Total swap = 11075580kB [94963.133129] 39321462 pages RAM [94963.134061] 0 pages HighMem/MovableOnly [94963.135073] 905848 pages reserved [94963.136003] 0 pages hwpoisoned [95114.207006] ll_ost_io01_039: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [95114.208962] ll_ost_io00_029: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [95114.210105] CPU: 7 PID: 527143 Comm: ll_ost_io01_039 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [95114.210108] ,mems_allowed=0 [95114.210111] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [95114.210517] ll_ost_io01_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [95114.212027] ll_ost_io02_019: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [95114.213321] ll_ost_io04_025: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [95114.215273] Call Trace: [95114.225791] dump_stack+0x41/0x60 [95114.226592] warn_alloc.cold.127+0x7b/0x108 [95114.227538] __alloc_pages_slowpath+0xcb2/0xcd0 [95114.228534] ? __alloc_pages_nodemask+0x166/0x330 [95114.229541] __alloc_pages_nodemask+0x2e2/0x330 [95114.230516] kmalloc_order+0x28/0x90 [95114.231339] kmalloc_order_trace+0x1d/0xb0 [95114.232267] __kmalloc+0x203/0x250 [95114.233047] virtqueue_add+0x493/0xc70 [95114.233897] ? finish_wait+0x80/0x80 [95114.234705] virtqueue_add_sgs+0x80/0xa0 [95114.235574] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [95114.236470] ll_ost_io04_043: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [95114.236479] ll_ost_io04_005: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [95114.236682] ? scsi_alloc_sgtables+0x84/0x1c0 [95114.238432] ,cpuset= [95114.240759] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [95114.241657] / [95114.242316] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [95114.243278] ,mems_allowed=0 [95114.243854] scsi_queue_rq+0x512/0xb10 [95114.245649] __blk_mq_try_issue_directly+0x163/0x200 [95114.248004] blk_mq_request_issue_directly+0x4e/0xb0 [95114.249039] blk_mq_try_issue_list_directly+0x62/0x100 [95114.250092] blk_mq_sched_insert_requests+0xa4/0xf0 [95114.251105] blk_mq_flush_plug_list+0x135/0x220 [95114.252067] blk_flush_plug_list+0xd7/0x100 [95114.252978] blk_finish_plug+0x25/0x36 [95114.253855] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [95114.255153] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [95114.256335] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.257614] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.258857] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [95114.259983] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [95114.261078] ofd_commitrw+0x5b4/0xd20 [ofd] [95114.262009] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.263122] obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.264224] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [95114.265325] ? set_next_entity+0xb5/0x1e0 [95114.266206] ? internal_add_timer+0x42/0x70 [95114.267105] ? _cond_resched+0x15/0x30 [95114.267925] ? mutex_lock+0xe/0x30 [95114.268688] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [95114.269816] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [95114.271096] ? finish_wait+0x80/0x80 [95114.271913] ptlrpc_main+0xbec/0x1530 [ptlrpc] [95114.272938] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [95114.274062] kthread+0x134/0x150 [95114.274781] ? set_kthread_struct+0x50/0x50 [95114.275661] ret_from_fork+0x1f/0x40 [95114.276450] warn_alloc_show_mem: 2 callbacks suppressed [95114.276450] CPU: 16 PID: 527165 Comm: ll_ost_io04_005 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [95114.276452] Mem-Info: [95114.277501] active_anon:27828 inactive_anon:252837 isolated_anon:0 active_file:27910509 inactive_file:6956204 isolated_file:11 unevictable:0 dirty:452 writeback:0 slab_reclaimable:921558 slab_unreclaimable:603859 mapped:22115 shmem:146 pagetables:5623 bounce:0 free:1339299 free_pcp:415 free_cma:0 [95114.279937] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [95114.280626] Node 0 active_anon:111312kB inactive_anon:1011348kB active_file:111642036kB inactive_file:27824816kB unevictable:0kB isolated(anon):0kB isolated(file):44kB mapped:88460kB dirty:1808kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35264kB pagetables:22492kB all_unreclaimable? no [95114.287243] Call Trace: [95114.288999] Node 0 [95114.294624] dump_stack+0x41/0x60 [95114.295290] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [95114.295931] warn_alloc.cold.127+0x7b/0x108 [95114.296751] lowmem_reserve[]: [95114.300783] __alloc_pages_slowpath+0xcb2/0xcd0 [95114.301737] 0 [95114.302496] __alloc_pages_nodemask+0x2e2/0x330 [95114.303505] 913 [95114.304058] kmalloc_order+0x28/0x90 [95114.305068] 150007 [95114.305660] kmalloc_order_trace+0x1d/0xb0 [95114.306514] 150007 [95114.307140] __kmalloc+0x203/0x250 [95114.308109] 150007 [95114.308758] ? __wake_up_common+0x7a/0x190 [95114.310258] virtqueue_add+0x493/0xc70 [95114.311235] Node 0 [95114.311788] ? finish_wait+0x80/0x80 [95114.312697] DMA32 free:596560kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:424kB active_file:176kB inactive_file:172kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [95114.313347] virtqueue_add_sgs+0x80/0xa0 [95114.314232] lowmem_reserve[]: [95114.318605] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [95114.319530] 0 [95114.320321] ? scsi_alloc_sgtables+0x84/0x1c0 [95114.321488] 0 [95114.322066] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [95114.323069] 149093 [95114.323649] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [95114.324774] 149093 [95114.325445] scsi_queue_rq+0x512/0xb10 [95114.326671] 149093 [95114.327360] __blk_mq_try_issue_directly+0x163/0x200 [95114.328985] blk_mq_request_issue_directly+0x4e/0xb0 [95114.330101] Node 0 [95114.330667] blk_mq_try_issue_list_directly+0x62/0x100 [95114.331764] Normal free:4741592kB min:49268kB low:201940kB high:354612kB active_anon:110932kB inactive_anon:1010924kB active_file:111650892kB inactive_file:27824640kB unevictable:0kB writepending:1808kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1544kB local_pcp:92kB free_cma:0kB [95114.332403] blk_mq_sched_insert_requests+0xa4/0xf0 [95114.333523] lowmem_reserve[]: [95114.338415] blk_mq_flush_plug_list+0x135/0x220 [95114.339512] 0 [95114.340299] blk_flush_plug_list+0xd7/0x100 [95114.341321] 0 [95114.341885] blk_finish_plug+0x25/0x36 [95114.342850] 0 [95114.343417] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [95114.344296] 0 [95114.344860] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [95114.346104] 0 [95114.346700] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.348391] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.349707] Node 0 [95114.350247] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [95114.351513] DMA: [95114.352141] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [95114.353260] 0*4kB [95114.353860] ofd_commitrw+0x5b4/0xd20 [ofd] [95114.354908] 0*8kB [95114.355542] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.356472] 0*16kB [95114.357099] obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.358123] 0*32kB [95114.358762] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [95114.359759] 0*64kB [95114.360424] ? flush_work+0x42/0x1d0 [95114.361489] 0*128kB [95114.362135] ? internal_add_timer+0x42/0x70 [95114.363006] 0*256kB [95114.363640] ? _cond_resched+0x15/0x30 [95114.364597] 0*512kB [95114.365250] ? mutex_lock+0xe/0x30 [95114.366157] 1*1024kB [95114.366812] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [95114.367635] (U) [95114.368337] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [95114.369412] 1*2048kB [95114.370025] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [95114.371228] (M) [95114.371910] ptlrpc_main+0xbec/0x1530 [ptlrpc] [95114.373145] 2*4096kB [95114.373779] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [95114.374754] (M) [95114.375439] kthread+0x134/0x150 [95114.376535] = 11264kB [95114.377137] ? set_kthread_struct+0x50/0x50 [95114.377958] Node 0 [95114.378622] ret_from_fork+0x1f/0x40 [95114.379579] DMA32: [95114.380188] CPU: 0 PID: 527259 Comm: ll_ost_io00_029 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [95114.380984] 58*4kB [95114.381623] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [95114.384188] (UMH) [95114.384742] Call Trace: [95114.386624] 69*8kB [95114.387194] dump_stack+0x41/0x60 [95114.387898] (UMH) [95114.388445] warn_alloc.cold.127+0x7b/0x108 [95114.389317] 72*16kB [95114.389860] __alloc_pages_slowpath+0xcb2/0xcd0 [95114.390840] (UMEH) [95114.391391] ? __alloc_pages_nodemask+0x166/0x330 [95114.392404] 57*32kB [95114.392930] __alloc_pages_nodemask+0x2e2/0x330 [95114.393954] (UMEH) [95114.394484] kmalloc_order+0x28/0x90 [95114.395472] 95*64kB [95114.395978] kmalloc_order_trace+0x1d/0xb0 [95114.396817] (UMH) [95114.397339] __kmalloc+0x203/0x250 [95114.398266] 92*128kB [95114.398762] virtqueue_add+0x493/0xc70 [95114.399558] (UM) [95114.400085] ? finish_wait+0x80/0x80 [95114.400964] 85*256kB [95114.401455] virtqueue_add_sgs+0x80/0xa0 [95114.402296] (UMH) [95114.402829] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [95114.403734] 85*512kB [95114.404236] ? scsi_alloc_sgtables+0x84/0x1c0 [95114.405374] (UMH) [95114.405921] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [95114.406909] 70*1024kB [95114.407431] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [95114.408537] (UMEH) [95114.409195] scsi_queue_rq+0x512/0xb10 [95114.410358] 54*2048kB [95114.410905] __blk_mq_try_issue_directly+0x163/0x200 [95114.411800] (UM) [95114.412368] blk_mq_request_issue_directly+0x4e/0xb0 [95114.413457] 80*4096kB [95114.413991] blk_mq_try_issue_list_directly+0x62/0x100 [95114.415080] (UM) [95114.415644] blk_mq_sched_insert_requests+0xa4/0xf0 [95114.416755] = 596848kB [95114.417260] blk_mq_flush_plug_list+0x135/0x220 [95114.418333] Node 0 [95114.418890] blk_flush_plug_list+0xd7/0x100 [95114.419898] Normal: [95114.420429] blk_finish_plug+0x25/0x36 [95114.421386] 839827*4kB [95114.421933] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [95114.422812] (UME) [95114.423385] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [95114.424593] 148985*8kB [95114.425107] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.426218] (UME) [95114.426798] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.428076] 10828*16kB [95114.428579] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [95114.429812] (UMEH) [95114.430387] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [95114.431482] 16*32kB [95114.431991] ofd_commitrw+0x5b4/0xd20 [ofd] [95114.433009] (UH) [95114.433533] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.434424] 0*64kB [95114.434918] obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.435908] 0*128kB [95114.436435] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [95114.437378] 0*256kB [95114.437908] ? newidle_balance+0x2b6/0x3b0 [95114.438869] 0*512kB [95114.439420] ? internal_add_timer+0x42/0x70 [95114.440324] 0*1024kB [95114.440852] ? _cond_resched+0x15/0x30 [95114.441813] 0*2048kB [95114.442344] ? mutex_lock+0xe/0x30 [95114.443190] 0*4096kB [95114.443739] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [95114.444512] = 4724948kB [95114.445049] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [95114.446076] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [95114.446634] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [95114.447785] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [95114.449234] ? finish_wait+0x80/0x80 [95114.450453] 34872751 total pagecache pages [95114.451840] ptlrpc_main+0xbec/0x1530 [ptlrpc] [95114.452657] 1389 pages in swap cache [95114.453466] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [95114.454427] Swap cache stats: add 1495705, delete 1494336, find 72753/87633 [95114.455160] kthread+0x134/0x150 [95114.456239] Free swap = 5690004kB [95114.457444] ? set_kthread_struct+0x50/0x50 [95114.458249] Total swap = 11075580kB [95114.458950] ret_from_fork+0x1f/0x40 [95114.459901] 39321462 pages RAM [95114.460631] CPU: 18 PID: 527680 Comm: ll_ost_io04_043 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [95114.461947] 0 pages HighMem/MovableOnly [95114.463117] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [95114.465800] 905848 pages reserved [95114.466639] Call Trace: [95114.468496] 0 pages hwpoisoned [95114.469259] dump_stack+0x41/0x60 [95114.471629] warn_alloc.cold.127+0x7b/0x108 [95114.472688] __alloc_pages_slowpath+0xcb2/0xcd0 [95114.473749] ? __alloc_pages_nodemask+0x166/0x330 [95114.474936] __alloc_pages_nodemask+0x2e2/0x330 [95114.476100] kmalloc_order+0x28/0x90 [95114.477068] kmalloc_order_trace+0x1d/0xb0 [95114.478140] __kmalloc+0x203/0x250 [95114.479074] virtqueue_add+0x493/0xc70 [95114.480083] ? finish_wait+0x80/0x80 [95114.481045] virtqueue_add_sgs+0x80/0xa0 [95114.482081] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [95114.483447] ? scsi_alloc_sgtables+0x84/0x1c0 [95114.484593] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [95114.485864] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [95114.487288] scsi_queue_rq+0x512/0xb10 [95114.488329] __blk_mq_try_issue_directly+0x163/0x200 [95114.489586] blk_mq_request_issue_directly+0x4e/0xb0 [95114.490850] blk_mq_try_issue_list_directly+0x62/0x100 [95114.492157] blk_mq_sched_insert_requests+0xa4/0xf0 [95114.493407] blk_mq_flush_plug_list+0x135/0x220 [95114.494572] blk_flush_plug_list+0xd7/0x100 [95114.495663] blk_finish_plug+0x25/0x36 [95114.496529] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [95114.497584] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [95114.498516] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.499620] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.500903] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [95114.502197] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [95114.503391] ofd_commitrw+0x5b4/0xd20 [ofd] [95114.504490] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.505834] obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.507113] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [95114.508423] ? flush_work+0x42/0x1d0 [95114.509396] ? internal_add_timer+0x42/0x70 [95114.510480] ? _cond_resched+0x15/0x30 [95114.511485] ? mutex_lock+0xe/0x30 [95114.512398] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [95114.513788] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [95114.515354] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [95114.516905] ptlrpc_main+0xbec/0x1530 [ptlrpc] [95114.518182] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [95114.519585] kthread+0x134/0x150 [95114.520494] ? set_kthread_struct+0x50/0x50 [95114.521589] ret_from_fork+0x1f/0x40 [95114.522550] CPU: 17 PID: 527240 Comm: ll_ost_io04_025 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [95114.524941] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [95114.526424] Call Trace: [95114.526954] dump_stack+0x41/0x60 [95114.527608] warn_alloc.cold.127+0x7b/0x108 [95114.528375] __alloc_pages_slowpath+0xcb2/0xcd0 [95114.529229] __alloc_pages_nodemask+0x2e2/0x330 [95114.530142] kmalloc_order+0x28/0x90 [95114.530883] kmalloc_order_trace+0x1d/0xb0 [95114.531637] __kmalloc+0x203/0x250 [95114.532452] virtqueue_add+0x493/0xc70 [95114.533281] ? finish_wait+0x80/0x80 [95114.533993] virtqueue_add_sgs+0x80/0xa0 [95114.534861] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [95114.535983] ? scsi_alloc_sgtables+0x84/0x1c0 [95114.536864] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [95114.538005] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [95114.539282] scsi_queue_rq+0x512/0xb10 [95114.540007] __blk_mq_try_issue_directly+0x163/0x200 [95114.540919] blk_mq_request_issue_directly+0x4e/0xb0 [95114.541789] blk_mq_try_issue_list_directly+0x62/0x100 [95114.542686] blk_mq_sched_insert_requests+0xa4/0xf0 [95114.543545] blk_mq_flush_plug_list+0x135/0x220 [95114.544410] blk_flush_plug_list+0xd7/0x100 [95114.545196] blk_finish_plug+0x25/0x36 [95114.545968] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [95114.547009] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [95114.547950] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.549048] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.550143] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [95114.551089] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [95114.551972] ofd_commitrw+0x5b4/0xd20 [ofd] [95114.552764] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.553691] obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.554582] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [95114.555525] ? flush_work+0x42/0x1d0 [95114.556238] ? internal_add_timer+0x42/0x70 [95114.557059] ? _cond_resched+0x15/0x30 [95114.557791] ? mutex_lock+0xe/0x30 [95114.558534] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [95114.559613] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [95114.560755] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [95114.561882] ptlrpc_main+0xbec/0x1530 [ptlrpc] [95114.562829] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [95114.563799] kthread+0x134/0x150 [95114.564433] ? set_kthread_struct+0x50/0x50 [95114.565243] ret_from_fork+0x1f/0x40 [95114.565942] CPU: 11 PID: 527172 Comm: ll_ost_io02_019 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [95114.568889] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [95114.571055] Call Trace: [95114.571879] dump_stack+0x41/0x60 [95114.572839] warn_alloc.cold.127+0x7b/0x108 [95114.573963] __alloc_pages_slowpath+0xcb2/0xcd0 [95114.575147] ? __alloc_pages_nodemask+0x166/0x330 [95114.576376] __alloc_pages_nodemask+0x2e2/0x330 [95114.577591] kmalloc_order+0x28/0x90 [95114.578603] kmalloc_order_trace+0x1d/0xb0 [95114.579704] __kmalloc+0x203/0x250 [95114.580662] ? common_interrupt+0xa/0xf [95114.581722] virtqueue_add+0x493/0xc70 [95114.582743] ? finish_wait+0x80/0x80 [95114.583739] virtqueue_add_sgs+0x80/0xa0 [95114.584799] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [95114.586154] ? scsi_alloc_sgtables+0x84/0x1c0 [95114.587345] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [95114.588688] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [95114.590118] scsi_queue_rq+0x512/0xb10 [95114.591140] __blk_mq_try_issue_directly+0x163/0x200 [95114.592383] blk_mq_request_issue_directly+0x4e/0xb0 [95114.593568] blk_mq_try_issue_list_directly+0x62/0x100 [95114.594820] blk_mq_sched_insert_requests+0xa4/0xf0 [95114.596068] blk_mq_flush_plug_list+0x135/0x220 [95114.597242] blk_flush_plug_list+0xd7/0x100 [95114.598353] blk_finish_plug+0x25/0x36 [95114.599392] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [95114.600869] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [95114.602231] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.603832] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.605447] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [95114.606826] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [95114.608088] ofd_commitrw+0x5b4/0xd20 [ofd] [95114.609195] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.610525] obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.611867] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [95114.613238] ? flush_work+0x42/0x1d0 [95114.614260] ? internal_add_timer+0x42/0x70 [95114.615381] ? _cond_resched+0x15/0x30 [95114.616413] ? mutex_lock+0xe/0x30 [95114.617377] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [95114.618816] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [95114.620382] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [95114.621967] ? lu_context_exit+0x41/0x70 [obdclass] [95114.623306] ptlrpc_main+0xbec/0x1530 [ptlrpc] [95114.624580] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [95114.625974] kthread+0x134/0x150 [95114.626893] ? set_kthread_struct+0x50/0x50 [95114.627996] ret_from_fork+0x1f/0x40 [95114.628980] CPU: 6 PID: 527199 Comm: ll_ost_io01_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [95114.632152] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [95114.634373] Call Trace: [95114.635245] dump_stack+0x41/0x60 [95114.636247] warn_alloc.cold.127+0x7b/0x108 [95114.637811] __alloc_pages_slowpath+0xcb2/0xcd0 [95114.639068] ? __alloc_pages_nodemask+0x166/0x330 [95114.640336] __alloc_pages_nodemask+0x2e2/0x330 [95114.641556] kmalloc_order+0x28/0x90 [95114.642566] kmalloc_order_trace+0x1d/0xb0 [95114.643703] __kmalloc+0x203/0x250 [95114.644669] virtqueue_add+0x493/0xc70 [95114.645712] ? finish_wait+0x80/0x80 [95114.646714] virtqueue_add_sgs+0x80/0xa0 [95114.647805] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [95114.649158] ? scsi_alloc_sgtables+0x84/0x1c0 [95114.650293] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [95114.651567] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [95114.652959] scsi_queue_rq+0x512/0xb10 [95114.653983] __blk_mq_try_issue_directly+0x163/0x200 [95114.655287] blk_mq_request_issue_directly+0x4e/0xb0 [95114.656606] blk_mq_try_issue_list_directly+0x62/0x100 [95114.657953] blk_mq_sched_insert_requests+0xa4/0xf0 [95114.659231] blk_mq_flush_plug_list+0x135/0x220 [95114.660448] blk_flush_plug_list+0xd7/0x100 [95114.661587] blk_finish_plug+0x25/0x36 [95114.662609] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [95114.664083] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [95114.665456] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.667041] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [95114.668596] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [95114.669978] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [95114.671488] ofd_commitrw+0x5b4/0xd20 [ofd] [95114.672628] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.673962] obd_commitrw+0x1b6/0x370 [ptlrpc] [95114.675282] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [95114.676654] ? flush_work+0x42/0x1d0 [95114.677661] ? internal_add_timer+0x42/0x70 [95114.678768] ? _cond_resched+0x15/0x30 [95114.679859] ? mutex_lock+0xe/0x30 [95114.680861] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [95114.682275] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [95114.683815] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [95114.685390] ptlrpc_main+0xbec/0x1530 [ptlrpc] [95114.686729] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [95114.688181] kthread+0x134/0x150 [95114.689101] ? set_kthread_struct+0x50/0x50 [95114.690203] ret_from_fork+0x1f/0x40 [96070.523088] ll_ost_io02_047: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [96070.525075] ll_ost_io03_012: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [96070.525087] ll_ost_io04_023: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [96070.525099] CPU: 19 PID: 527238 Comm: ll_ost_io04_023 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96070.525107] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96070.525109] Call Trace: [96070.525139] dump_stack+0x41/0x60 [96070.525161] warn_alloc.cold.127+0x7b/0x108 [96070.525167] __alloc_pages_slowpath+0xcb2/0xcd0 [96070.525171] ? __alloc_pages_nodemask+0x166/0x330 [96070.525175] __alloc_pages_nodemask+0x2e2/0x330 [96070.525178] kmalloc_order+0x28/0x90 [96070.525185] kmalloc_order_trace+0x1d/0xb0 [96070.525188] __kmalloc+0x203/0x250 [96070.525194] virtqueue_add+0x493/0xc70 [96070.525200] ? finish_wait+0x80/0x80 [96070.525205] virtqueue_add_sgs+0x80/0xa0 [96070.525221] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96070.525239] ? scsi_alloc_sgtables+0x84/0x1c0 [96070.525245] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96070.525249] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96070.525253] scsi_queue_rq+0x512/0xb10 [96070.525256] __blk_mq_try_issue_directly+0x163/0x200 [96070.525262] blk_mq_request_issue_directly+0x4e/0xb0 [96070.525267] blk_mq_try_issue_list_directly+0x62/0x100 [96070.525271] blk_mq_sched_insert_requests+0xa4/0xf0 [96070.525275] blk_mq_flush_plug_list+0x135/0x220 [96070.525279] blk_flush_plug_list+0xd7/0x100 [96070.525285] blk_finish_plug+0x25/0x36 [96070.525299] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96070.525403] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96070.525513] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.525542] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.525571] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96070.525610] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96070.525676] ofd_commitrw+0x5b4/0xd20 [ofd] [96070.525695] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.525821] obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.525929] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96070.526047] ? flush_work+0x42/0x1d0 [96070.526054] ? internal_add_timer+0x42/0x70 [96070.526058] ? _cond_resched+0x15/0x30 [96070.526065] ? mutex_lock+0xe/0x30 [96070.526072] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96070.526179] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96070.526292] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96070.526395] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96070.526496] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96070.529027] ,cpuset= [96070.531907] kthread+0x134/0x150 [96070.534804] / [96070.535055] ll_ost_io02_025: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [96070.536964] ? set_kthread_struct+0x50/0x50 [96070.537887] ,mems_allowed=0 [96070.537889] /,mems_allowed=0 [96070.538139] ll_ost_io02_040: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [96070.538877] ret_from_fork+0x1f/0x40 [96070.541027] warn_alloc_show_mem: 6 callbacks suppressed [96070.541027] CPU: 9 PID: 527181 Comm: ll_ost_io02_025 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96070.541035] Mem-Info: [96070.541038] active_anon:27815 inactive_anon:219910 isolated_anon:0 active_file:27106312 inactive_file:7588361 isolated_file:5 unevictable:0 dirty:315 writeback:0 slab_reclaimable:997351 slab_unreclaimable:603419 mapped:21366 shmem:146 pagetables:5656 bounce:0 free:1468826 free_pcp:705 free_cma:0 [96070.541043] Node 0 active_anon:111260kB inactive_anon:879640kB active_file:108425248kB inactive_file:30353444kB unevictable:0kB isolated(anon):0kB isolated(file):20kB mapped:85464kB dirty:1260kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35472kB pagetables:22624kB all_unreclaimable? no [96070.541051] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [96070.541056] lowmem_reserve[]: 0 913 150007 150007 150007 [96070.541060] Node 0 DMA32 free:596508kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:28kB active_file:92kB inactive_file:760kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:952kB local_pcp:0kB free_cma:0kB [96070.541066] lowmem_reserve[]: 0 0 149093 149093 149093 [96070.541070] Node 0 Normal free:5267532kB min:49268kB low:201940kB high:354612kB active_anon:110880kB inactive_anon:879612kB active_file:108425192kB inactive_file:30352684kB unevictable:0kB writepending:1260kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1868kB local_pcp:112kB free_cma:0kB [96070.541075] lowmem_reserve[]: 0 0 0 0 0 [96070.541078] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [96070.541088] Node 0 DMA32: 19*4kB (M) 18*8kB (M) 16*16kB (ME) 18*32kB (ME) 62*64kB (UME) 63*128kB (UME) 61*256kB (UM) 63*512kB (UMEH) 53*1024kB (UM) 51*2048kB (UM) 92*4096kB (UM) = 596508kB [96070.541102] Node 0 Normal: 945580*4kB (UME) 182082*8kB (UME) 1729*16kB (UMEH) 1*32kB (U) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 5266672kB [96070.541114] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [96070.541115] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [96070.541117] 34694605 total pagecache pages [96070.541117] 1364 pages in swap cache [96070.541118] Swap cache stats: add 1554179, delete 1552838, find 77398/93203 [96070.541119] Free swap = 5494720kB [96070.541120] Total swap = 11075580kB [96070.541121] 39321462 pages RAM [96070.541121] 0 pages HighMem/MovableOnly [96070.541122] 905848 pages reserved [96070.541122] 0 pages hwpoisoned [96070.575663] ll_ost_io00_064: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [96070.575675] ll_ost_io00_018: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [96070.576348] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96070.577568] ,cpuset= [96070.578880] Call Trace: [96070.580188] / [96070.581221] dump_stack+0x41/0x60 [96070.582375] ,mems_allowed=0 [96070.583040] warn_alloc.cold.127+0x7b/0x108 [96070.584467] __alloc_pages_slowpath+0xcb2/0xcd0 [96070.658335] ? __alloc_pages_nodemask+0x166/0x330 [96070.659444] __alloc_pages_nodemask+0x2e2/0x330 [96070.660516] kmalloc_order+0x28/0x90 [96070.661413] kmalloc_order_trace+0x1d/0xb0 [96070.662377] __kmalloc+0x203/0x250 [96070.663238] virtqueue_add+0x493/0xc70 [96070.664149] ? finish_wait+0x80/0x80 [96070.665072] virtqueue_add_sgs+0x80/0xa0 [96070.666042] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96070.667228] ? scsi_alloc_sgtables+0x84/0x1c0 [96070.668247] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96070.669355] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96070.670553] scsi_queue_rq+0x512/0xb10 [96070.671432] __blk_mq_try_issue_directly+0x163/0x200 [96070.672509] blk_mq_request_issue_directly+0x4e/0xb0 [96070.673597] blk_mq_try_issue_list_directly+0x62/0x100 [96070.674700] blk_mq_sched_insert_requests+0xa4/0xf0 [96070.675752] blk_mq_flush_plug_list+0x135/0x220 [96070.676757] blk_flush_plug_list+0xd7/0x100 [96070.677725] blk_finish_plug+0x25/0x36 [96070.678590] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96070.679820] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96070.680942] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.682222] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.683509] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96070.684634] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96070.685661] ofd_commitrw+0x5b4/0xd20 [ofd] [96070.686573] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.687661] obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.688727] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96070.689819] ? flush_work+0x42/0x1d0 [96070.690627] ? internal_add_timer+0x42/0x70 [96070.691512] ? _cond_resched+0x15/0x30 [96070.692322] ? mutex_lock+0xe/0x30 [96070.693078] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96070.694189] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96070.695419] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96070.696676] ? finish_wait+0x80/0x80 [96070.697466] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96070.698516] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96070.699653] kthread+0x134/0x150 [96070.700410] ? set_kthread_struct+0x50/0x50 [96070.701280] ret_from_fork+0x1f/0x40 [96070.702055] CPU: 1 PID: 527757 Comm: ll_ost_io00_064 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96070.704707] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96070.706527] Call Trace: [96070.707157] dump_stack+0x41/0x60 [96070.707941] warn_alloc.cold.127+0x7b/0x108 [96070.708901] __alloc_pages_slowpath+0xcb2/0xcd0 [96070.709901] ? __alloc_pages_nodemask+0x166/0x330 [96070.710916] __alloc_pages_nodemask+0x2e2/0x330 [96070.711896] kmalloc_order+0x28/0x90 [96070.712718] kmalloc_order_trace+0x1d/0xb0 [96070.713643] __kmalloc+0x203/0x250 [96070.714435] virtqueue_add+0x493/0xc70 [96070.715323] ? finish_wait+0x80/0x80 [96070.716165] virtqueue_add_sgs+0x80/0xa0 [96070.717047] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96070.718176] ? scsi_alloc_sgtables+0x84/0x1c0 [96070.719131] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96070.720207] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96070.721403] scsi_queue_rq+0x512/0xb10 [96070.722264] __blk_mq_try_issue_directly+0x163/0x200 [96070.723322] blk_mq_request_issue_directly+0x4e/0xb0 [96070.724382] blk_mq_try_issue_list_directly+0x62/0x100 [96070.725471] blk_mq_sched_insert_requests+0xa4/0xf0 [96070.726515] blk_mq_flush_plug_list+0x135/0x220 [96070.727525] blk_flush_plug_list+0xd7/0x100 [96070.728476] blk_finish_plug+0x25/0x36 [96070.729346] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96070.730588] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96070.731727] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.733059] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.734346] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96070.735464] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96070.736525] ofd_commitrw+0x5b4/0xd20 [ofd] [96070.737469] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.738587] obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.739666] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96070.740781] ? set_next_entity+0xb5/0x1e0 [96070.741709] ? internal_add_timer+0x42/0x70 [96070.742661] ? _cond_resched+0x15/0x30 [96070.743512] ? mutex_lock+0xe/0x30 [96070.744312] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96070.745504] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96070.746827] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96070.748141] ? finish_wait+0x80/0x80 [96070.748977] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96070.750075] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96070.751287] kthread+0x134/0x150 [96070.752078] ? set_kthread_struct+0x50/0x50 [96070.753019] ret_from_fork+0x1f/0x40 [96070.753891] CPU: 11 PID: 527253 Comm: ll_ost_io02_047 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96070.756989] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96070.759201] Call Trace: [96070.759950] dump_stack+0x41/0x60 [96070.760833] warn_alloc.cold.127+0x7b/0x108 [96070.761882] __alloc_pages_slowpath+0xcb2/0xcd0 [96070.763009] ? __alloc_pages_nodemask+0x166/0x330 [96070.764158] __alloc_pages_nodemask+0x2e2/0x330 [96070.765301] kmalloc_order+0x28/0x90 [96070.766261] kmalloc_order_trace+0x1d/0xb0 [96070.767306] __kmalloc+0x203/0x250 [96070.768230] ? __wake_up_common+0x7a/0x190 [96070.769115] virtqueue_add+0x493/0xc70 [96070.770052] ? finish_wait+0x80/0x80 [96070.770872] virtqueue_add_sgs+0x80/0xa0 [96070.771764] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96070.772754] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96070.773675] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96070.774714] scsi_queue_rq+0x512/0xb10 [96070.775411] __blk_mq_try_issue_directly+0x163/0x200 [96070.776296] blk_mq_request_issue_directly+0x4e/0xb0 [96070.777174] blk_mq_try_issue_list_directly+0x62/0x100 [96070.778084] blk_mq_sched_insert_requests+0xa4/0xf0 [96070.778932] blk_mq_flush_plug_list+0x135/0x220 [96070.779750] blk_flush_plug_list+0xd7/0x100 [96070.780522] blk_finish_plug+0x25/0x36 [96070.781256] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96070.782286] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [96070.783504] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.784955] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.786132] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96070.787200] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96070.788379] ofd_commitrw+0x5b4/0xd20 [ofd] [96070.789442] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.790727] obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.791838] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96070.793073] ? flush_work+0x42/0x1d0 [96070.793999] ? internal_add_timer+0x42/0x70 [96070.795011] ? _cond_resched+0x15/0x30 [96070.795988] ? mutex_lock+0xe/0x30 [96070.796913] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96070.798297] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96070.799801] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96070.801293] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96070.802363] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96070.803739] kthread+0x134/0x150 [96070.804634] ? set_kthread_struct+0x50/0x50 [96070.805678] ret_from_fork+0x1f/0x40 [96070.806509] CPU: 13 PID: 527032 Comm: ll_ost_io03_012 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96070.809835] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96070.812377] Call Trace: [96070.813143] dump_stack+0x41/0x60 [96070.814066] warn_alloc.cold.127+0x7b/0x108 [96070.815203] __alloc_pages_slowpath+0xcb2/0xcd0 [96070.816190] ? __alloc_pages_nodemask+0x166/0x330 [96070.817286] __alloc_pages_nodemask+0x2e2/0x330 [96070.818382] kmalloc_order+0x28/0x90 [96070.819129] kmalloc_order_trace+0x1d/0xb0 [96070.819944] __kmalloc+0x203/0x250 [96070.820995] virtqueue_add+0x493/0xc70 [96070.821779] ? finish_wait+0x80/0x80 [96070.822635] virtqueue_add_sgs+0x80/0xa0 [96070.823634] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96070.824787] ? scsi_alloc_sgtables+0x84/0x1c0 [96070.825887] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96070.827031] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96070.828160] scsi_queue_rq+0x512/0xb10 [96070.829134] __blk_mq_try_issue_directly+0x163/0x200 [96070.830193] blk_mq_request_issue_directly+0x4e/0xb0 [96070.831404] blk_mq_try_issue_list_directly+0x62/0x100 [96070.832404] blk_mq_sched_insert_requests+0xa4/0xf0 [96070.833568] blk_mq_flush_plug_list+0x135/0x220 [96070.834504] blk_flush_plug_list+0xd7/0x100 [96070.835316] blk_finish_plug+0x25/0x36 [96070.836104] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96070.837492] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96070.838823] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.840156] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.841228] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96070.842156] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96070.843029] ofd_commitrw+0x5b4/0xd20 [ofd] [96070.843819] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.844772] obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.845679] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96070.846601] ? flush_work+0x42/0x1d0 [96070.847297] ? internal_add_timer+0x42/0x70 [96070.848062] ? _cond_resched+0x15/0x30 [96070.848761] ? mutex_lock+0xe/0x30 [96070.849437] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96070.850411] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96070.851484] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96070.852562] ? finish_wait+0x80/0x80 [96070.853251] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96070.854124] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96070.855097] kthread+0x134/0x150 [96070.855735] ? set_kthread_struct+0x50/0x50 [96070.856496] ret_from_fork+0x1f/0x40 [96070.857177] CPU: 8 PID: 527245 Comm: ll_ost_io02_040 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96070.860258] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96070.862359] Call Trace: [96070.863134] dump_stack+0x41/0x60 [96070.864054] warn_alloc.cold.127+0x7b/0x108 [96070.865161] __alloc_pages_slowpath+0xcb2/0xcd0 [96070.866343] ? __alloc_pages_nodemask+0x166/0x330 [96070.867535] __alloc_pages_nodemask+0x2e2/0x330 [96070.868709] kmalloc_order+0x28/0x90 [96070.869727] kmalloc_order_trace+0x1d/0xb0 [96070.870797] __kmalloc+0x203/0x250 [96070.871718] ? __wake_up_common+0x7a/0x190 [96070.872770] virtqueue_add+0x493/0xc70 [96070.873776] ? finish_wait+0x80/0x80 [96070.874728] virtqueue_add_sgs+0x80/0xa0 [96070.875746] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96070.876885] ? scsi_alloc_sgtables+0x84/0x1c0 [96070.877888] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96070.879174] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96070.880546] scsi_queue_rq+0x512/0xb10 [96070.881541] __blk_mq_try_issue_directly+0x163/0x200 [96070.882748] blk_mq_request_issue_directly+0x4e/0xb0 [96070.883968] blk_mq_try_issue_list_directly+0x62/0x100 [96070.885217] blk_mq_sched_insert_requests+0xa4/0xf0 [96070.886419] blk_mq_flush_plug_list+0x135/0x220 [96070.887561] blk_flush_plug_list+0xd7/0x100 [96070.888631] blk_finish_plug+0x25/0x36 [96070.889638] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96070.891101] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.892689] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.894255] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96070.895646] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96070.896877] ofd_commitrw+0x5b4/0xd20 [ofd] [96070.897971] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.899266] obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.900563] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96070.901882] ? common_interrupt+0xa/0xf [96070.902880] ? flush_work+0x42/0x1d0 [96070.903858] ? internal_add_timer+0x42/0x70 [96070.904960] ? _cond_resched+0x15/0x30 [96070.905939] ? mutex_lock+0xe/0x30 [96070.906739] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96070.908091] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96070.909555] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96070.911043] ? finish_wait+0x80/0x80 [96070.912021] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96070.913119] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96070.914478] kthread+0x134/0x150 [96070.915331] ? set_kthread_struct+0x50/0x50 [96070.916155] ret_from_fork+0x1f/0x40 [96070.917044] CPU: 0 PID: 527189 Comm: ll_ost_io00_018 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96070.919841] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96070.921570] Call Trace: [96070.922118] dump_stack+0x41/0x60 [96070.922758] warn_alloc.cold.127+0x7b/0x108 [96070.923545] __alloc_pages_slowpath+0xcb2/0xcd0 [96070.924489] ? __alloc_pages_nodemask+0x166/0x330 [96070.925641] __alloc_pages_nodemask+0x2e2/0x330 [96070.926720] kmalloc_order+0x28/0x90 [96070.927477] kmalloc_order_trace+0x1d/0xb0 [96070.928326] __kmalloc+0x203/0x250 [96070.929035] virtqueue_add+0x493/0xc70 [96070.929812] ? finish_wait+0x80/0x80 [96070.930570] virtqueue_add_sgs+0x80/0xa0 [96070.931376] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96070.932432] ? scsi_alloc_sgtables+0x84/0x1c0 [96070.933310] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96070.934355] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96070.935673] scsi_queue_rq+0x512/0xb10 [96070.936473] __blk_mq_try_issue_directly+0x163/0x200 [96070.937431] blk_mq_request_issue_directly+0x4e/0xb0 [96070.938372] blk_mq_try_issue_list_directly+0x62/0x100 [96070.939364] blk_mq_sched_insert_requests+0xa4/0xf0 [96070.940296] blk_mq_flush_plug_list+0x135/0x220 [96070.941186] blk_flush_plug_list+0xd7/0x100 [96070.941974] blk_finish_plug+0x25/0x36 [96070.942701] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96070.943774] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96070.944812] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.945956] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96070.947087] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96070.948077] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96070.948951] ofd_commitrw+0x5b4/0xd20 [ofd] [96070.949763] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.950706] obd_commitrw+0x1b6/0x370 [ptlrpc] [96070.951625] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96070.952601] ? set_next_entity+0xb5/0x1e0 [96070.953399] ? internal_add_timer+0x42/0x70 [96070.954228] ? _cond_resched+0x15/0x30 [96070.955147] ? mutex_lock+0xe/0x30 [96070.955964] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96070.957099] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96070.958568] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96070.960046] ? finish_wait+0x80/0x80 [96070.960989] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96070.962101] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96070.963097] kthread+0x134/0x150 [96070.963741] ? set_kthread_struct+0x50/0x50 [96070.964492] ret_from_fork+0x1f/0x40 [96829.828384] ll_ost_io04_051: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [96829.831916] CPU: 17 PID: 527688 Comm: ll_ost_io04_051 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96829.834971] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96829.837088] Call Trace: [96829.837897] dump_stack+0x41/0x60 [96829.838826] warn_alloc.cold.127+0x7b/0x108 [96829.839952] __alloc_pages_slowpath+0xcb2/0xcd0 [96829.841133] __alloc_pages_nodemask+0x2e2/0x330 [96829.842336] kmalloc_order+0x28/0x90 [96829.843343] kmalloc_order_trace+0x1d/0xb0 [96829.844445] __kmalloc+0x203/0x250 [96829.845409] virtqueue_add+0x493/0xc70 [96829.846481] ? finish_wait+0x80/0x80 [96829.847505] virtqueue_add_sgs+0x80/0xa0 [96829.848573] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96829.849929] ? scsi_alloc_sgtables+0x84/0x1c0 [96829.851069] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96829.852338] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96829.853713] scsi_queue_rq+0x512/0xb10 [96829.854722] __blk_mq_try_issue_directly+0x163/0x200 [96829.855971] blk_mq_request_issue_directly+0x4e/0xb0 [96829.857215] blk_mq_try_issue_list_directly+0x62/0x100 [96829.858501] blk_mq_sched_insert_requests+0xa4/0xf0 [96829.859743] blk_mq_flush_plug_list+0x135/0x220 [96829.860903] blk_flush_plug_list+0xd7/0x100 [96829.862021] blk_finish_plug+0x25/0x36 [96829.863098] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96829.864559] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96829.865846] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96829.867393] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96829.868751] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96829.869919] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96829.871227] ofd_commitrw+0x5b4/0xd20 [ofd] [96829.872329] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96829.873632] obd_commitrw+0x1b6/0x370 [ptlrpc] [96829.874899] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96829.876228] ? newidle_balance+0x2b6/0x3b0 [96829.877323] ? internal_add_timer+0x42/0x70 [96829.878426] ? _cond_resched+0x15/0x30 [96829.879431] ? mutex_lock+0xe/0x30 [96829.880359] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96829.881778] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96829.883179] ll_ost_io01_013: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [96829.883334] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96829.886941] ? finish_wait+0x80/0x80 [96829.887922] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96829.889216] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96829.889618] ll_ost_io00_034: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [96829.890618] kthread+0x134/0x150 [96829.890625] ? set_kthread_struct+0x50/0x50 [96829.894288] ret_from_fork+0x1f/0x40 [96829.897032] warn_alloc_show_mem: 6 callbacks suppressed [96829.897032] CPU: 6 PID: 527066 Comm: ll_ost_io01_013 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96829.897044] Mem-Info: [96829.898092] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96829.900909] active_anon:27828 inactive_anon:205674 isolated_anon:0 active_file:31889850 inactive_file:3439847 isolated_file:1 unevictable:0 dirty:177 writeback:3 slab_reclaimable:1001356 slab_unreclaimable:604064 mapped:21257 shmem:146 pagetables:5680 bounce:0 free:842662 free_pcp:464 free_cma:0 [96829.901505] Call Trace: [96829.903628] Node 0 active_anon:111312kB inactive_anon:822696kB active_file:127559400kB inactive_file:13759388kB unevictable:0kB isolated(anon):0kB isolated(file):4kB mapped:85028kB dirty:708kB writeback:12kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35376kB pagetables:22720kB all_unreclaimable? no [96829.910406] dump_stack+0x41/0x60 [96829.911209] Node 0 [96829.917220] warn_alloc.cold.127+0x7b/0x108 [96829.918203] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [96829.918969] __alloc_pages_slowpath+0xcb2/0xcd0 [96829.920137] lowmem_reserve[]: [96829.925041] __alloc_pages_nodemask+0x2e2/0x330 [96829.926254] 0 [96829.927184] kmalloc_order+0x28/0x90 [96829.928457] 913 [96829.929001] kmalloc_order_trace+0x1d/0xb0 [96829.929910] 150007 [96829.930465] __kmalloc+0x203/0x250 [96829.931496] 150007 [96829.932138] ? __wake_up_common+0x7a/0x190 [96829.933054] 150007 [96829.933821] virtqueue_add+0x493/0xc70 [96829.935750] ? finish_wait+0x80/0x80 [96829.936838] Node 0 [96829.937495] virtqueue_add_sgs+0x80/0xa0 [96829.938529] DMA32 free:596512kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:28kB active_file:108kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [96829.939302] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96829.940449] lowmem_reserve[]: [96829.945726] ? scsi_alloc_sgtables+0x84/0x1c0 [96829.947014] 0 [96829.947993] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96829.949229] 0 [96829.949946] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96829.951294] 149093 [96829.952027] scsi_queue_rq+0x512/0xb10 [96829.953488] 149093 [96829.954260] __blk_mq_try_issue_directly+0x163/0x200 [96829.955339] 149093 [96829.956127] blk_mq_request_issue_directly+0x4e/0xb0 [96829.958271] blk_mq_try_issue_list_directly+0x62/0x100 [96829.959597] Node 0 [96829.960270] blk_mq_sched_insert_requests+0xa4/0xf0 [96829.961617] Normal free:2763632kB min:49268kB low:201940kB high:354612kB active_anon:110932kB inactive_anon:822668kB active_file:127559616kB inactive_file:13759388kB unevictable:0kB writepending:720kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1672kB local_pcp:24kB free_cma:0kB [96829.962418] blk_mq_flush_plug_list+0x135/0x220 [96829.963756] lowmem_reserve[]: [96829.969618] blk_flush_plug_list+0xd7/0x100 [96829.970837] 0 [96829.971763] blk_finish_plug+0x25/0x36 [96829.972903] 0 [96829.973574] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96829.974615] 0 [96829.975269] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96829.976584] 0 [96829.977255] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96829.978591] 0 [96829.979279] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96829.981265] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96829.982584] Node 0 [96829.983207] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96829.984493] DMA: [96829.985250] ofd_commitrw+0x5b4/0xd20 [ofd] [96829.986507] 0*4kB [96829.987218] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96829.988308] 0*8kB [96829.988977] obd_commitrw+0x1b6/0x370 [ptlrpc] [96829.990148] 0*16kB [96829.990767] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96829.991923] 0*32kB [96829.992658] ? flush_work+0x42/0x1d0 [96829.993907] 0*64kB [96829.994583] ? internal_add_timer+0x42/0x70 [96829.995597] 0*128kB [96829.996336] ? _cond_resched+0x15/0x30 [96829.997489] 0*256kB [96829.998263] ? mutex_lock+0xe/0x30 [96829.999325] 0*512kB [96830.000008] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96830.000970] 1*1024kB [96830.001715] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96830.002911] (U) [96830.003667] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96830.005083] 1*2048kB [96830.005618] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96830.007009] (M) [96830.007598] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96830.008744] 2*4096kB [96830.009403] kthread+0x134/0x150 [96830.010691] (M) [96830.011460] ? set_kthread_struct+0x50/0x50 [96830.012394] = 11264kB [96830.013086] ret_from_fork+0x1f/0x40 [96830.014188] Node 0 [96830.014962] CPU: 0 PID: 527273 Comm: ll_ost_io00_034 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [96830.016128] DMA32: [96830.016946] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [96830.020330] 26*4kB [96830.021192] Call Trace: [96830.023554] (UMH) [96830.024341] dump_stack+0x41/0x60 [96830.025291] 39*8kB [96830.026067] warn_alloc.cold.127+0x7b/0x108 [96830.027057] (UMH) [96830.027795] __alloc_pages_slowpath+0xcb2/0xcd0 [96830.028997] 28*16kB [96830.029668] ? __alloc_pages_nodemask+0x166/0x330 [96830.030984] (UMEH) [96830.031533] __alloc_pages_nodemask+0x2e2/0x330 [96830.032888] 37*32kB [96830.033553] kmalloc_order+0x28/0x90 [96830.034765] (UMH) [96830.035291] kmalloc_order_trace+0x1d/0xb0 [96830.036039] 48*64kB [96830.036566] __kmalloc+0x203/0x250 [96830.037414] (UMH) [96830.037949] ? __wake_up_common+0x7a/0x190 [96830.038919] 51*128kB [96830.039603] virtqueue_add+0x493/0xc70 [96830.040702] (UMH) [96830.041441] ? finish_wait+0x80/0x80 [96830.042493] 49*256kB [96830.043193] virtqueue_add_sgs+0x80/0xa0 [96830.044218] (UMH) [96830.044959] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [96830.046029] 51*512kB [96830.046718] ? scsi_alloc_sgtables+0x84/0x1c0 [96830.048065] (UME) [96830.048764] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [96830.049926] 42*1024kB [96830.050629] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [96830.051919] (UMH) [96830.052685] scsi_queue_rq+0x512/0xb10 [96830.054075] 44*2048kB [96830.054766] __blk_mq_try_issue_directly+0x163/0x200 [96830.055815] (UM) [96830.056544] blk_mq_request_issue_directly+0x4e/0xb0 [96830.057814] 101*4096kB [96830.058471] blk_mq_try_issue_list_directly+0x62/0x100 [96830.059727] (UM) [96830.060494] blk_mq_sched_insert_requests+0xa4/0xf0 [96830.061793] = 597120kB [96830.062330] blk_mq_flush_plug_list+0x135/0x220 [96830.063588] Node 0 [96830.064244] blk_flush_plug_list+0xd7/0x100 [96830.065423] Normal: [96830.066138] blk_finish_plug+0x25/0x36 [96830.067247] 434278*4kB [96830.067989] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [96830.068989] (UME) [96830.069758] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [96830.071158] 103705*8kB [96830.071850] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96830.073166] (UME) [96830.073797] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [96830.075283] 10702*16kB [96830.075817] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [96830.077277] (UMH) [96830.078014] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [96830.079295] 11*32kB [96830.079955] ofd_commitrw+0x5b4/0xd20 [ofd] [96830.081154] (H) [96830.081849] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [96830.082903] 3*64kB [96830.083535] obd_commitrw+0x1b6/0x370 [ptlrpc] [96830.084677] (H) [96830.085358] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [96830.086474] 0*128kB [96830.087121] ? newidle_balance+0x2b6/0x3b0 [96830.088363] 0*256kB [96830.089102] ? internal_add_timer+0x42/0x70 [96830.090203] 0*512kB [96830.090925] ? _cond_resched+0x15/0x30 [96830.092049] 0*1024kB [96830.092762] ? mutex_lock+0xe/0x30 [96830.093805] 0*2048kB [96830.094527] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [96830.095456] 0*4096kB [96830.096077] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [96830.097306] = 2738528kB [96830.098032] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [96830.099374] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [96830.100012] ? finish_wait+0x80/0x80 [96830.101422] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [96830.103331] ptlrpc_main+0xbec/0x1530 [ptlrpc] [96830.104304] 35342649 total pagecache pages [96830.106201] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [96830.107337] 1484 pages in swap cache [96830.108373] kthread+0x134/0x150 [96830.109651] Swap cache stats: add 1589064, delete 1587616, find 81127/97769 [96830.110619] ? set_kthread_struct+0x50/0x50 [96830.111561] Free swap = 5387836kB [96830.113164] ret_from_fork+0x1f/0x40 [96830.114300] Total swap = 11075580kB [96830.117308] 39321462 pages RAM [96830.118235] 0 pages HighMem/MovableOnly [96830.119359] 905848 pages reserved [96830.120422] 0 pages hwpoisoned [97112.784332] ll_ost_io04_058: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97112.784354] ll_ost_io00_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97112.784997] ll_ost_io00_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97112.785013] CPU: 0 PID: 527755 Comm: ll_ost_io00_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97112.785023] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97112.785025] Call Trace: [97112.785067] dump_stack+0x41/0x60 [97112.785096] warn_alloc.cold.127+0x7b/0x108 [97112.785104] __alloc_pages_slowpath+0xcb2/0xcd0 [97112.785113] ? __alloc_pages_nodemask+0x166/0x330 [97112.785116] __alloc_pages_nodemask+0x2e2/0x330 [97112.785119] kmalloc_order+0x28/0x90 [97112.785134] kmalloc_order_trace+0x1d/0xb0 [97112.785142] __kmalloc+0x203/0x250 [97112.785148] virtqueue_add+0x493/0xc70 [97112.785156] ? finish_wait+0x80/0x80 [97112.785165] virtqueue_add_sgs+0x80/0xa0 [97112.785184] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97112.785210] ? scsi_alloc_sgtables+0x84/0x1c0 [97112.785216] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97112.785224] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97112.785229] scsi_queue_rq+0x512/0xb10 [97112.785233] __blk_mq_try_issue_directly+0x163/0x200 [97112.785244] blk_mq_request_issue_directly+0x4e/0xb0 [97112.785249] blk_mq_try_issue_list_directly+0x62/0x100 [97112.785257] blk_mq_sched_insert_requests+0xa4/0xf0 [97112.785262] blk_mq_flush_plug_list+0x135/0x220 [97112.785267] blk_flush_plug_list+0xd7/0x100 [97112.785278] blk_finish_plug+0x25/0x36 [97112.785299] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97112.785435] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97112.785578] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97112.785615] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97112.785657] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97112.785697] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97112.785787] ofd_commitrw+0x5b4/0xd20 [ofd] [97112.785811] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97112.786004] obd_commitrw+0x1b6/0x370 [ptlrpc] [97112.786171] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97112.786340] ? flush_work+0x42/0x1d0 [97112.786347] ? internal_add_timer+0x42/0x70 [97112.786355] ? _cond_resched+0x15/0x30 [97112.786364] ? mutex_lock+0xe/0x30 [97112.786372] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97112.786533] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97112.786689] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97112.786838] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97112.786961] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97112.787082] kthread+0x134/0x150 [97112.787087] ? set_kthread_struct+0x50/0x50 [97112.787091] ret_from_fork+0x1f/0x40 [97112.787097] warn_alloc_show_mem: 2 callbacks suppressed [97112.787104] Mem-Info: [97112.787107] active_anon:27937 inactive_anon:199100 isolated_anon:0 active_file:30345576 inactive_file:4696303 isolated_file:33 unevictable:0 dirty:1144 writeback:0 slab_reclaimable:1001144 slab_unreclaimable:605375 mapped:21207 shmem:146 pagetables:5732 bounce:0 free:1138341 free_pcp:461 free_cma:0 [97112.787114] Node 0 active_anon:111748kB inactive_anon:796400kB active_file:121382304kB inactive_file:18785212kB unevictable:0kB isolated(anon):0kB isolated(file):132kB mapped:84828kB dirty:4576kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35520kB pagetables:22928kB all_unreclaimable? no [97112.787124] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97112.787131] lowmem_reserve[]: 0 913 150007 150007 150007 [97112.787137] Node 0 DMA32 free:596468kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:12kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97112.787147] lowmem_reserve[]: 0 0 149093 149093 149093 [97112.787151] Node 0 Normal free:3945312kB min:49268kB low:201940kB high:354612kB active_anon:111368kB inactive_anon:796376kB active_file:121381548kB inactive_file:18785572kB unevictable:0kB writepending:4576kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1840kB local_pcp:128kB free_cma:0kB [97112.787158] lowmem_reserve[]: 0 0 0 0 0 [97112.787162] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [97112.787178] Node 0 DMA32: 21*4kB (UM) 24*8kB (UMH) 26*16kB (UME) 42*32kB (UM) 44*64kB (UMH) 47*128kB (UM) 45*256kB (UMH) 47*512kB (UME) 39*1024kB (UM) 43*2048kB (UM) 103*4096kB (UM) = 596340kB [97112.787200] Node 0 Normal: 612014*4kB (UME) 173147*8kB (UME) 7060*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 3946192kB [97112.787217] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97112.787220] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97112.787222] 35042280 total pagecache pages [97112.787223] 2023 pages in swap cache [97112.787224] Swap cache stats: add 1603641, delete 1601641, find 82512/99425 [97112.787226] Free swap = 5339756kB [97112.787227] Total swap = 11075580kB [97112.787229] 39321462 pages RAM [97112.787229] 0 pages HighMem/MovableOnly [97112.787230] 905848 pages reserved [97112.787231] 0 pages hwpoisoned [97112.787799] CPU: 19 PID: 527695 Comm: ll_ost_io04_058 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97112.790709] ,cpuset= [97112.790731] ll_ost_io03_066: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97112.793854] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97112.796975] / [97112.798975] ll_ost_io04_029: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97112.799311] Call Trace: [97112.800256] / [97112.800256] ,mems_allowed=0 [97112.801343] dump_stack+0x41/0x60 [97112.802599] ,mems_allowed=0 [97112.803915] warn_alloc.cold.127+0x7b/0x108 [97112.807579] __alloc_pages_slowpath+0xcb2/0xcd0 [97112.816486] ll_ost_io02_003: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97112.816936] ? __alloc_pages_nodemask+0x166/0x330 [97112.818141] ,cpuset= [97112.819729] __alloc_pages_nodemask+0x2e2/0x330 [97112.820625] / [97112.821266] ll_ost_io03_082: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97112.821966] kmalloc_order+0x28/0x90 [97112.823043] ,mems_allowed=0 [97112.823044] /,mems_allowed=0 [97112.824420] kmalloc_order_trace+0x1d/0xb0 [97112.826973] __kmalloc+0x203/0x250 [97112.841113] ll_ost_io00_075: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97112.842109] virtqueue_add+0x493/0xc70 [97112.843116] ,cpuset= [97112.844277] ? finish_wait+0x80/0x80 [97112.845292] / [97112.846269] virtqueue_add_sgs+0x80/0xa0 [97112.847375] ,mems_allowed=0 [97112.848844] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97112.851106] ? scsi_alloc_sgtables+0x84/0x1c0 [97112.950169] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97112.951450] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97112.952837] scsi_queue_rq+0x512/0xb10 [97112.953887] __blk_mq_try_issue_directly+0x163/0x200 [97112.955165] blk_mq_request_issue_directly+0x4e/0xb0 [97112.956405] blk_mq_try_issue_list_directly+0x62/0x100 [97112.957512] blk_mq_sched_insert_requests+0xa4/0xf0 [97112.958749] blk_mq_flush_plug_list+0x135/0x220 [97112.959909] blk_flush_plug_list+0xd7/0x100 [97112.961010] blk_finish_plug+0x25/0x36 [97112.962036] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97112.963492] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97112.964843] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97112.966412] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97112.967842] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97112.969078] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97112.970353] ofd_commitrw+0x5b4/0xd20 [ofd] [97112.971473] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97112.972817] obd_commitrw+0x1b6/0x370 [ptlrpc] [97112.974119] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97112.975433] ? flush_work+0x42/0x1d0 [97112.976399] ? internal_add_timer+0x42/0x70 [97112.977472] ? _cond_resched+0x15/0x30 [97112.978474] ? mutex_lock+0xe/0x30 [97112.979429] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97112.980471] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97112.981574] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97112.982661] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97112.983552] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97112.984529] kthread+0x134/0x150 [97112.985170] ? set_kthread_struct+0x50/0x50 [97112.985955] ret_from_fork+0x1f/0x40 [97112.986648] CPU: 2 PID: 527758 Comm: ll_ost_io00_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97112.989775] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97112.992230] Call Trace: [97112.993066] dump_stack+0x41/0x60 [97112.994025] warn_alloc.cold.127+0x7b/0x108 [97112.995158] __alloc_pages_slowpath+0xcb2/0xcd0 [97112.996354] ? __alloc_pages_nodemask+0x166/0x330 [97112.997690] __alloc_pages_nodemask+0x2e2/0x330 [97112.998958] kmalloc_order+0x28/0x90 [97112.999981] kmalloc_order_trace+0x1d/0xb0 [97113.001043] __kmalloc+0x203/0x250 [97113.001936] virtqueue_add+0x493/0xc70 [97113.002773] ? finish_wait+0x80/0x80 [97113.003484] virtqueue_add_sgs+0x80/0xa0 [97113.004250] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97113.005365] ? scsi_alloc_sgtables+0x84/0x1c0 [97113.006442] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97113.007644] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97113.008961] scsi_queue_rq+0x512/0xb10 [97113.009930] __blk_mq_try_issue_directly+0x163/0x200 [97113.011113] blk_mq_request_issue_directly+0x4e/0xb0 [97113.012294] blk_mq_try_issue_list_directly+0x62/0x100 [97113.013513] blk_mq_sched_insert_requests+0xa4/0xf0 [97113.014688] blk_mq_flush_plug_list+0x135/0x220 [97113.015786] blk_flush_plug_list+0xd7/0x100 [97113.016832] blk_finish_plug+0x25/0x36 [97113.017799] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97113.019195] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97113.020485] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.021979] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.023451] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97113.024724] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97113.025909] ofd_commitrw+0x5b4/0xd20 [ofd] [97113.026956] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.028037] obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.028914] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97113.029808] ? flush_work+0x42/0x1d0 [97113.030474] ? internal_add_timer+0x42/0x70 [97113.031246] ? _cond_resched+0x15/0x30 [97113.032043] ? mutex_lock+0xe/0x30 [97113.032742] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97113.034042] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97113.035517] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97113.037032] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97113.038275] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97113.039638] kthread+0x134/0x150 [97113.040515] ? set_kthread_struct+0x50/0x50 [97113.041545] ret_from_fork+0x1f/0x40 [97113.042475] CPU: 16 PID: 527255 Comm: ll_ost_io04_029 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97113.044933] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97113.046620] Call Trace: [97113.047191] dump_stack+0x41/0x60 [97113.047857] warn_alloc.cold.127+0x7b/0x108 [97113.048707] __alloc_pages_slowpath+0xcb2/0xcd0 [97113.049578] ? __alloc_pages_nodemask+0x166/0x330 [97113.050473] __alloc_pages_nodemask+0x2e2/0x330 [97113.051291] kmalloc_order+0x28/0x90 [97113.052003] kmalloc_order_trace+0x1d/0xb0 [97113.052833] __kmalloc+0x203/0x250 [97113.053535] ? __wake_up_common+0x7a/0x190 [97113.054300] virtqueue_add+0x493/0xc70 [97113.055001] ? finish_wait+0x80/0x80 [97113.055674] virtqueue_add_sgs+0x80/0xa0 [97113.056421] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97113.057349] ? scsi_alloc_sgtables+0x84/0x1c0 [97113.058152] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97113.059048] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97113.060040] scsi_queue_rq+0x512/0xb10 [97113.060782] __blk_mq_try_issue_directly+0x163/0x200 [97113.061670] blk_mq_request_issue_directly+0x4e/0xb0 [97113.062541] blk_mq_try_issue_list_directly+0x62/0x100 [97113.063419] blk_mq_sched_insert_requests+0xa4/0xf0 [97113.064290] blk_mq_flush_plug_list+0x135/0x220 [97113.065104] blk_flush_plug_list+0xd7/0x100 [97113.065868] blk_finish_plug+0x25/0x36 [97113.066565] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97113.067573] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97113.068524] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.069627] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.070700] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97113.071634] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97113.072511] ofd_commitrw+0x5b4/0xd20 [ofd] [97113.073295] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.074230] obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.075144] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97113.076055] ? flush_work+0x42/0x1d0 [97113.076750] ? internal_add_timer+0x42/0x70 [97113.077514] ? _cond_resched+0x15/0x30 [97113.078224] ? mutex_lock+0xe/0x30 [97113.078929] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97113.079999] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97113.081125] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97113.082257] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97113.083159] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97113.084142] kthread+0x134/0x150 [97113.084785] ? set_kthread_struct+0x50/0x50 [97113.085560] ret_from_fork+0x1f/0x40 [97113.086252] CPU: 15 PID: 527555 Comm: ll_ost_io03_082 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97113.089106] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97113.091153] Call Trace: [97113.091912] dump_stack+0x41/0x60 [97113.092824] warn_alloc.cold.127+0x7b/0x108 [97113.093901] __alloc_pages_slowpath+0xcb2/0xcd0 [97113.095059] ? __alloc_pages_nodemask+0x166/0x330 [97113.096292] __alloc_pages_nodemask+0x2e2/0x330 [97113.097471] kmalloc_order+0x28/0x90 [97113.098456] kmalloc_order_trace+0x1d/0xb0 [97113.099581] __kmalloc+0x203/0x250 [97113.100545] virtqueue_add+0x493/0xc70 [97113.101621] ? finish_wait+0x80/0x80 [97113.102597] virtqueue_add_sgs+0x80/0xa0 [97113.103614] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97113.104932] ? scsi_alloc_sgtables+0x84/0x1c0 [97113.106089] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97113.107424] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97113.108858] scsi_queue_rq+0x512/0xb10 [97113.109932] __blk_mq_try_issue_directly+0x163/0x200 [97113.111220] blk_mq_request_issue_directly+0x4e/0xb0 [97113.112481] blk_mq_try_issue_list_directly+0x62/0x100 [97113.113745] blk_mq_sched_insert_requests+0xa4/0xf0 [97113.114940] blk_mq_flush_plug_list+0x135/0x220 [97113.116105] blk_flush_plug_list+0xd7/0x100 [97113.117207] blk_finish_plug+0x25/0x36 [97113.118198] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97113.119659] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97113.121002] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.122552] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.124070] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97113.125391] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97113.126619] ofd_commitrw+0x5b4/0xd20 [ofd] [97113.127702] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.129034] obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.130244] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97113.131528] ? set_next_entity+0xb5/0x1e0 [97113.132571] ? internal_add_timer+0x42/0x70 [97113.133665] ? _cond_resched+0x15/0x30 [97113.134667] ? mutex_lock+0xe/0x30 [97113.135581] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97113.136795] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97113.137950] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97113.139279] ? finish_wait+0x80/0x80 [97113.140240] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97113.141479] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97113.142820] kthread+0x134/0x150 [97113.143695] ? set_kthread_struct+0x50/0x50 [97113.144789] ret_from_fork+0x1f/0x40 [97113.145765] CPU: 12 PID: 527451 Comm: ll_ost_io03_066 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97113.148636] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97113.150617] Call Trace: [97113.151269] dump_stack+0x41/0x60 [97113.152168] warn_alloc.cold.127+0x7b/0x108 [97113.153228] __alloc_pages_slowpath+0xcb2/0xcd0 [97113.154363] ? __alloc_pages_nodemask+0x166/0x330 [97113.155524] __alloc_pages_nodemask+0x2e2/0x330 [97113.156654] kmalloc_order+0x28/0x90 [97113.157640] kmalloc_order_trace+0x1d/0xb0 [97113.158711] __kmalloc+0x203/0x250 [97113.159634] ? __wake_up_common+0x7a/0x190 [97113.160715] virtqueue_add+0x493/0xc70 [97113.161682] ? finish_wait+0x80/0x80 [97113.162417] virtqueue_add_sgs+0x80/0xa0 [97113.163374] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97113.164509] ? scsi_alloc_sgtables+0x84/0x1c0 [97113.165410] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97113.166600] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97113.167866] scsi_queue_rq+0x512/0xb10 [97113.168785] __blk_mq_try_issue_directly+0x163/0x200 [97113.169734] blk_mq_request_issue_directly+0x4e/0xb0 [97113.170680] blk_mq_try_issue_list_directly+0x62/0x100 [97113.171874] blk_mq_sched_insert_requests+0xa4/0xf0 [97113.173082] blk_mq_flush_plug_list+0x135/0x220 [97113.174216] blk_flush_plug_list+0xd7/0x100 [97113.175118] blk_finish_plug+0x25/0x36 [97113.176072] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97113.177134] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97113.178301] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.179807] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.181258] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97113.182527] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97113.183710] ofd_commitrw+0x5b4/0xd20 [ofd] [97113.184784] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.186105] obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.187357] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97113.188637] ? set_next_entity+0xb5/0x1e0 [97113.189674] ? internal_add_timer+0x42/0x70 [97113.190740] ? _cond_resched+0x15/0x30 [97113.191734] ? mutex_lock+0xe/0x30 [97113.192577] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97113.193672] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97113.194789] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97113.195980] ? finish_wait+0x80/0x80 [97113.196757] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97113.197717] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97113.198770] kthread+0x134/0x150 [97113.199440] ? set_kthread_struct+0x50/0x50 [97113.200507] ret_from_fork+0x1f/0x40 [97113.201265] CPU: 0 PID: 527772 Comm: ll_ost_io00_075 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97113.203415] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97113.204854] Call Trace: [97113.205379] dump_stack+0x41/0x60 [97113.206035] warn_alloc.cold.127+0x7b/0x108 [97113.206799] __alloc_pages_slowpath+0xcb2/0xcd0 [97113.207622] __alloc_pages_nodemask+0x2e2/0x330 [97113.208512] kmalloc_order+0x28/0x90 [97113.209188] kmalloc_order_trace+0x1d/0xb0 [97113.209931] __kmalloc+0x203/0x250 [97113.210584] virtqueue_add+0x493/0xc70 [97113.211281] ? finish_wait+0x80/0x80 [97113.211940] virtqueue_add_sgs+0x80/0xa0 [97113.212664] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97113.213571] ? scsi_alloc_sgtables+0x84/0x1c0 [97113.214381] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97113.215276] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97113.216250] scsi_queue_rq+0x512/0xb10 [97113.216971] __blk_mq_try_issue_directly+0x163/0x200 [97113.217832] blk_mq_request_issue_directly+0x4e/0xb0 [97113.218695] blk_mq_try_issue_list_directly+0x62/0x100 [97113.219599] blk_mq_sched_insert_requests+0xa4/0xf0 [97113.220480] blk_mq_flush_plug_list+0x135/0x220 [97113.221277] blk_flush_plug_list+0xd7/0x100 [97113.222029] blk_finish_plug+0x25/0x36 [97113.222730] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97113.223745] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97113.224685] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.225756] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.226811] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97113.227747] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97113.228621] ofd_commitrw+0x5b4/0xd20 [ofd] [97113.229378] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.230323] obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.231252] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97113.232165] ? newidle_balance+0x2b6/0x3b0 [97113.232909] ? internal_add_timer+0x42/0x70 [97113.233660] ? _cond_resched+0x15/0x30 [97113.234367] ? mutex_lock+0xe/0x30 [97113.235027] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97113.235983] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97113.237034] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97113.238086] ? finish_wait+0x80/0x80 [97113.238781] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97113.239672] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97113.240614] kthread+0x134/0x150 [97113.241240] ? set_kthread_struct+0x50/0x50 [97113.241992] ret_from_fork+0x1f/0x40 [97113.242695] CPU: 10 PID: 527004 Comm: ll_ost_io02_003 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97113.245868] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97113.248156] Call Trace: [97113.248984] dump_stack+0x41/0x60 [97113.249970] warn_alloc.cold.127+0x7b/0x108 [97113.251111] __alloc_pages_slowpath+0xcb2/0xcd0 [97113.252306] ? __alloc_pages_nodemask+0x166/0x330 [97113.253530] __alloc_pages_nodemask+0x2e2/0x330 [97113.254727] kmalloc_order+0x28/0x90 [97113.255712] kmalloc_order_trace+0x1d/0xb0 [97113.256824] __kmalloc+0x203/0x250 [97113.257793] virtqueue_add+0x493/0xc70 [97113.258794] ? finish_wait+0x80/0x80 [97113.259763] virtqueue_add_sgs+0x80/0xa0 [97113.260856] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97113.262179] ? scsi_alloc_sgtables+0x84/0x1c0 [97113.263305] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97113.264548] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97113.265977] scsi_queue_rq+0x512/0xb10 [97113.267014] __blk_mq_try_issue_directly+0x163/0x200 [97113.268255] blk_mq_request_issue_directly+0x4e/0xb0 [97113.269358] blk_mq_try_issue_list_directly+0x62/0x100 [97113.270452] blk_mq_sched_insert_requests+0xa4/0xf0 [97113.271661] blk_mq_flush_plug_list+0x135/0x220 [97113.272810] blk_flush_plug_list+0xd7/0x100 [97113.273905] blk_finish_plug+0x25/0x36 [97113.274965] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97113.276356] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97113.277423] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.278941] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97113.280454] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97113.281806] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97113.283077] ofd_commitrw+0x5b4/0xd20 [ofd] [97113.284201] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.285513] obd_commitrw+0x1b6/0x370 [ptlrpc] [97113.286791] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97113.288133] ? flush_work+0x42/0x1d0 [97113.289155] ? internal_add_timer+0x42/0x70 [97113.290261] ? _cond_resched+0x15/0x30 [97113.291297] ? mutex_lock+0xe/0x30 [97113.292266] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97113.293707] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97113.295261] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97113.296805] ? osd_it_ea_store+0x20/0x20 [osd_ldiskfs] [97113.298104] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97113.299385] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97113.300814] kthread+0x134/0x150 [97113.301756] ? set_kthread_struct+0x50/0x50 [97113.302866] ret_from_fork+0x1f/0x40 [97149.016884] ll_ost_io02_038: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97149.016900] ll_ost_io01_037: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97149.016904] ll_ost_io04_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97149.016908] ll_ost_io01_081: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97149.016912] / [97149.016911] ll_ost_io04_079: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97149.016913] ,mems_allowed=0 [97149.016914] ,cpuset= [97149.016914] ,cpuset= [97149.016916] /,mems_allowed=0 [97149.016917] / [97149.016916] ll_ost_io02_078: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97149.016917] CPU: 17 PID: 527193 Comm: ll_ost_io04_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.016917] ll_ost_io04_078: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97149.016919] ,mems_allowed=0 [97149.016921] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.016921] ,cpuset= [97149.016923] Call Trace: [97149.016923] / [97149.016923] ,cpuset= [97149.016925] ,mems_allowed=0 [97149.016926] / [97149.016925] ll_ost_io04_021: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97149.016927] ,mems_allowed=0 [97149.016928] ,cpuset= [97149.016929] /,mems_allowed=0 [97149.016928] dump_stack+0x41/0x60 [97149.016936] warn_alloc.cold.127+0x7b/0x108 [97149.016943] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.016946] ? virtqueue_add+0x959/0xc70 [97149.016952] __alloc_pages_nodemask+0x2e2/0x330 [97149.016955] kmalloc_order+0x28/0x90 [97149.016960] kmalloc_order_trace+0x1d/0xb0 [97149.016963] __kmalloc+0x203/0x250 [97149.016968] virtqueue_add+0x493/0xc70 [97149.016972] ? finish_wait+0x80/0x80 [97149.016976] virtqueue_add_sgs+0x80/0xa0 [97149.016981] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.016986] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.016991] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.016994] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.016997] scsi_queue_rq+0x512/0xb10 [97149.017000] __blk_mq_try_issue_directly+0x163/0x200 [97149.017006] blk_mq_request_issue_directly+0x4e/0xb0 [97149.017010] blk_mq_try_issue_list_directly+0x62/0x100 [97149.017013] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.017017] blk_mq_flush_plug_list+0x135/0x220 [97149.017021] blk_flush_plug_list+0xd7/0x100 [97149.017025] blk_finish_plug+0x25/0x36 [97149.017030] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.017068] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.017102] ll_ost_io03_046: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97149.017099] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.017124] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.017142] ll_ost_io00_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97149.017151] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.017177] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.017196] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.017213] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.017332] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.017438] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.017541] ? flush_work+0x42/0x1d0 [97149.017546] ? internal_add_timer+0x42/0x70 [97149.017549] ? _cond_resched+0x15/0x30 [97149.017553] ? mutex_lock+0xe/0x30 [97149.017555] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.017666] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.017771] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.017875] ? finish_wait+0x80/0x80 [97149.017878] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.017980] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.018080] kthread+0x134/0x150 [97149.018084] ? set_kthread_struct+0x50/0x50 [97149.018087] ret_from_fork+0x1f/0x40 [97149.018092] warn_alloc_show_mem: 7 callbacks suppressed [97149.018094] Mem-Info: [97149.018093] CPU: 16 PID: 528067 Comm: ll_ost_io04_079 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.018096] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.018097] Call Trace: [97149.018097] active_anon:27936 inactive_anon:199284 isolated_anon:0 active_file:31433071 inactive_file:3697810 isolated_file:32 unevictable:0 dirty:700 writeback:0 slab_reclaimable:1001097 slab_unreclaimable:605796 mapped:21670 shmem:146 pagetables:5692 bounce:0 free:1047808 free_pcp:440 free_cma:0 [97149.018099] dump_stack+0x41/0x60 [97149.018103] Node 0 active_anon:111744kB inactive_anon:797136kB active_file:125732284kB inactive_file:14791240kB unevictable:0kB isolated(anon):0kB isolated(file):128kB mapped:86680kB dirty:2800kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35440kB pagetables:22768kB all_unreclaimable? no [97149.018104] warn_alloc.cold.127+0x7b/0x108 [97149.018109] Node 0 [97149.018110] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.018110] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97149.018113] ? __alloc_pages_nodemask+0x166/0x330 [97149.018116] lowmem_reserve[]: 0 913 [97149.018117] __alloc_pages_nodemask+0x2e2/0x330 [97149.018118] 150007 150007 150007 [97149.018122] Node 0 [97149.018120] kmalloc_order+0x28/0x90 [97149.018122] DMA32 free:596436kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:12kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97149.018125] kmalloc_order_trace+0x1d/0xb0 [97149.018127] lowmem_reserve[]: 0 0 149093 149093 149093 [97149.018129] __kmalloc+0x203/0x250 [97149.018132] Node 0 Normal free:3583016kB min:49268kB low:201940kB high:354612kB active_anon:111364kB inactive_anon:797132kB active_file:125732424kB inactive_file:14791540kB unevictable:0kB writepending:2800kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1792kB local_pcp:156kB free_cma:0kB [97149.018133] virtqueue_add+0x493/0xc70 [97149.018138] lowmem_reserve[]: 0 0 0 [97149.018139] ? finish_wait+0x80/0x80 [97149.018141] 0 0 [97149.018143] Node 0 [97149.018142] virtqueue_add_sgs+0x80/0xa0 [97149.018143] DMA: 0*4kB 0*8kB 0*16kB 0*32kB [97149.018147] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.018149] 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB [97149.018152] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.018155] (M) 2*4096kB (M) = 11264kB [97149.018157] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.018158] Node 0 DMA32: 21*4kB (UM) [97149.018161] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.018162] 24*8kB (UMH) 26*16kB (UME) [97149.018164] scsi_queue_rq+0x512/0xb10 [97149.018166] 33*32kB (UMH) 44*64kB (UMH) 47*128kB (UMH) [97149.018168] __blk_mq_try_issue_directly+0x163/0x200 [97149.018170] 44*256kB (UM) 48*512kB (UMEH) 39*1024kB [97149.018173] blk_mq_request_issue_directly+0x4e/0xb0 [97149.018175] (UM) 43*2048kB (UM) 103*4096kB (UM) [97149.018177] blk_mq_try_issue_list_directly+0x62/0x100 [97149.018179] = 596308kB [97149.018180] Node 0 Normal: 336300*4kB [97149.018181] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.018183] (UME) 258872*8kB (UME) 10460*16kB (UMEH) [97149.018185] blk_mq_flush_plug_list+0x135/0x220 [97149.018187] 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB [97149.018189] blk_flush_plug_list+0xd7/0x100 [97149.018192] 0*2048kB 0*4096kB = 3583536kB [97149.018195] blk_finish_plug+0x25/0x36 [97149.018196] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97149.018198] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97149.018199] 35130967 total pagecache pages [97149.018200] 1522 pages in swap cache [97149.018201] Swap cache stats: add 1603907, delete 1602408, find 82710/99661 [97149.018203] Free swap = 5340268kB [97149.018204] Total swap = 11075580kB [97149.018205] 39321462 pages RAM [97149.018206] 0 pages HighMem/MovableOnly [97149.018206] 905848 pages reserved [97149.018207] 0 pages hwpoisoned [97149.018199] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.018234] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.018267] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.018296] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.018324] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.018351] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.018371] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.018382] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.018492] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.018565] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.018632] ? flush_work+0x42/0x1d0 [97149.018637] ? internal_add_timer+0x42/0x70 [97149.018640] ? _cond_resched+0x15/0x30 [97149.018643] ? mutex_lock+0xe/0x30 [97149.018645] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.018714] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.018783] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.018853] ? finish_wait+0x80/0x80 [97149.018856] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.018925] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.018992] kthread+0x134/0x150 [97149.018995] ? set_kthread_struct+0x50/0x50 [97149.018998] ret_from_fork+0x1f/0x40 [97149.019003] CPU: 18 PID: 528066 Comm: ll_ost_io04_078 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.019007] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.019008] Call Trace: [97149.019012] dump_stack+0x41/0x60 [97149.019020] warn_alloc.cold.127+0x7b/0x108 [97149.019026] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.019029] ? __alloc_pages_nodemask+0x166/0x330 [97149.019033] __alloc_pages_nodemask+0x2e2/0x330 [97149.019036] kmalloc_order+0x28/0x90 [97149.019041] kmalloc_order_trace+0x1d/0xb0 [97149.019044] __kmalloc+0x203/0x250 [97149.019049] virtqueue_add+0x493/0xc70 [97149.019055] ? finish_wait+0x80/0x80 [97149.019059] virtqueue_add_sgs+0x80/0xa0 [97149.019064] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.019070] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.019076] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.019079] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.019082] scsi_queue_rq+0x512/0xb10 [97149.019086] __blk_mq_try_issue_directly+0x163/0x200 [97149.019092] blk_mq_request_issue_directly+0x4e/0xb0 [97149.019096] blk_mq_try_issue_list_directly+0x62/0x100 [97149.019100] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.019104] blk_mq_flush_plug_list+0x135/0x220 [97149.019108] blk_flush_plug_list+0xd7/0x100 [97149.019113] blk_finish_plug+0x25/0x36 [97149.019119] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.021732] ,cpuset= [97149.023951] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.026421] / [97149.027024] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.029459] ,mems_allowed=0 [97149.030245] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.031711] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.241769] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.242793] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.243707] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.244805] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.245852] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.246936] ? flush_work+0x42/0x1d0 [97149.247731] ? internal_add_timer+0x42/0x70 [97149.248617] ? _cond_resched+0x15/0x30 [97149.249448] ? mutex_lock+0xe/0x30 [97149.250224] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.251354] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.252587] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.253829] ? finish_wait+0x80/0x80 [97149.254625] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.255667] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.256800] kthread+0x134/0x150 [97149.257526] ? set_kthread_struct+0x50/0x50 [97149.258389] ret_from_fork+0x1f/0x40 [97149.259174] CPU: 19 PID: 527236 Comm: ll_ost_io04_021 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.261867] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.263879] Call Trace: [97149.264612] dump_stack+0x41/0x60 [97149.265505] warn_alloc.cold.127+0x7b/0x108 [97149.266618] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.267755] ? __alloc_pages_nodemask+0x166/0x330 [97149.268925] __alloc_pages_nodemask+0x2e2/0x330 [97149.270053] kmalloc_order+0x28/0x90 [97149.271006] kmalloc_order_trace+0x1d/0xb0 [97149.272079] __kmalloc+0x203/0x250 [97149.273014] ? __wake_up_common+0x7a/0x190 [97149.274062] virtqueue_add+0x493/0xc70 [97149.275074] ? finish_wait+0x80/0x80 [97149.276031] virtqueue_add_sgs+0x80/0xa0 [97149.277039] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.278330] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.279451] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.280712] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.282089] scsi_queue_rq+0x512/0xb10 [97149.283118] __blk_mq_try_issue_directly+0x163/0x200 [97149.284363] blk_mq_request_issue_directly+0x4e/0xb0 [97149.285587] blk_mq_try_issue_list_directly+0x62/0x100 [97149.286840] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.288060] blk_mq_flush_plug_list+0x135/0x220 [97149.289218] blk_flush_plug_list+0xd7/0x100 [97149.290286] blk_finish_plug+0x25/0x36 [97149.291288] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.292744] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.294052] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.295550] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.297033] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.298331] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.299537] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.300615] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.301926] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.303192] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.304481] ? flush_work+0x42/0x1d0 [97149.305432] ? internal_add_timer+0x42/0x70 [97149.306495] ? _cond_resched+0x15/0x30 [97149.307467] ? mutex_lock+0xe/0x30 [97149.308369] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.309720] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.311195] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.312753] ? finish_wait+0x80/0x80 [97149.313734] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.314966] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.316302] kthread+0x134/0x150 [97149.317170] ? set_kthread_struct+0x50/0x50 [97149.318240] ret_from_fork+0x1f/0x40 [97149.319190] CPU: 1 PID: 527174 Comm: ll_ost_io00_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.321912] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.323808] Call Trace: [97149.324482] dump_stack+0x41/0x60 [97149.325454] warn_alloc.cold.127+0x7b/0x108 [97149.326439] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.327523] ? __alloc_pages_nodemask+0x166/0x330 [97149.328594] __alloc_pages_nodemask+0x2e2/0x330 [97149.329691] kmalloc_order+0x28/0x90 [97149.330586] kmalloc_order_trace+0x1d/0xb0 [97149.331591] __kmalloc+0x203/0x250 [97149.332424] virtqueue_add+0x493/0xc70 [97149.333400] virtqueue_add_sgs+0x80/0xa0 [97149.334734] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.335973] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.337332] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.338515] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.339796] scsi_queue_rq+0x512/0xb10 [97149.340710] __blk_mq_try_issue_directly+0x163/0x200 [97149.341864] blk_mq_request_issue_directly+0x4e/0xb0 [97149.344266] blk_mq_try_issue_list_directly+0x62/0x100 [97149.345462] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.346604] blk_mq_flush_plug_list+0x135/0x220 [97149.348035] blk_flush_plug_list+0xd7/0x100 [97149.349044] blk_finish_plug+0x25/0x36 [97149.349933] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.351217] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.352405] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.353780] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.355132] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.356318] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.357404] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.358363] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.359520] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.360648] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.361836] ? flush_work+0x42/0x1d0 [97149.362696] ? internal_add_timer+0x42/0x70 [97149.363631] ? _cond_resched+0x15/0x30 [97149.364505] ? mutex_lock+0xe/0x30 [97149.365312] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.366525] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.367880] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.369235] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.370397] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.371592] kthread+0x134/0x150 [97149.372364] ? set_kthread_struct+0x50/0x50 [97149.373303] ret_from_fork+0x1f/0x40 [97149.374138] CPU: 10 PID: 527243 Comm: ll_ost_io02_038 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.376519] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.378007] Call Trace: [97149.378557] dump_stack+0x41/0x60 [97149.379208] warn_alloc.cold.127+0x7b/0x108 [97149.379973] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.380792] ? __alloc_pages_nodemask+0x166/0x330 [97149.381619] __alloc_pages_nodemask+0x2e2/0x330 [97149.382423] kmalloc_order+0x28/0x90 [97149.383104] kmalloc_order_trace+0x1d/0xb0 [97149.383851] __kmalloc+0x203/0x250 [97149.384504] virtqueue_add+0x493/0xc70 [97149.385222] ? finish_wait+0x80/0x80 [97149.385898] virtqueue_add_sgs+0x80/0xa0 [97149.386620] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.387549] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.388332] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.389205] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.390180] scsi_queue_rq+0x512/0xb10 [97149.390896] __blk_mq_try_issue_directly+0x163/0x200 [97149.391760] blk_mq_request_issue_directly+0x4e/0xb0 [97149.392622] blk_mq_try_issue_list_directly+0x62/0x100 [97149.393502] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.394357] blk_mq_flush_plug_list+0x135/0x220 [97149.395184] blk_flush_plug_list+0xd7/0x100 [97149.395964] blk_finish_plug+0x25/0x36 [97149.396685] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.397700] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.398649] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.399731] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.400786] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.401804] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.402912] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.403698] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.404613] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.405496] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.406400] ? flush_work+0x42/0x1d0 [97149.407080] ? internal_add_timer+0x42/0x70 [97149.407839] ? _cond_resched+0x15/0x30 [97149.408545] ? mutex_lock+0xe/0x30 [97149.409186] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.410141] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.411226] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.412299] ? tgt_key_exit+0xc/0x20 [ptlrpc] [97149.413157] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.414046] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.415003] kthread+0x134/0x150 [97149.415644] ? set_kthread_struct+0x50/0x50 [97149.416391] ret_from_fork+0x1f/0x40 [97149.417082] CPU: 5 PID: 527141 Comm: ll_ost_io01_037 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.419966] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.422039] Call Trace: [97149.422805] dump_stack+0x41/0x60 [97149.423728] warn_alloc.cold.127+0x7b/0x108 [97149.424770] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.425925] ? __alloc_pages_nodemask+0x166/0x330 [97149.427140] __alloc_pages_nodemask+0x2e2/0x330 [97149.428293] kmalloc_order+0x28/0x90 [97149.429277] kmalloc_order_trace+0x1d/0xb0 [97149.430383] __kmalloc+0x203/0x250 [97149.431335] virtqueue_add+0x493/0xc70 [97149.432346] ? finish_wait+0x80/0x80 [97149.433325] virtqueue_add_sgs+0x80/0xa0 [97149.434377] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.435712] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.436867] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.438165] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.439562] scsi_queue_rq+0x512/0xb10 [97149.440585] __blk_mq_try_issue_directly+0x163/0x200 [97149.441855] blk_mq_request_issue_directly+0x4e/0xb0 [97149.443133] blk_mq_try_issue_list_directly+0x62/0x100 [97149.444422] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.445686] blk_mq_flush_plug_list+0x135/0x220 [97149.446897] blk_flush_plug_list+0xd7/0x100 [97149.448004] blk_finish_plug+0x25/0x36 [97149.449012] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.450446] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.451905] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.453534] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.455051] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.456105] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.456987] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.457750] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.458678] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.459573] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.460484] ? flush_work+0x42/0x1d0 [97149.461165] ? internal_add_timer+0x42/0x70 [97149.461935] ? _cond_resched+0x15/0x30 [97149.462636] ? mutex_lock+0xe/0x30 [97149.463307] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.464351] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.465456] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.466555] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.467426] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.468387] kthread+0x134/0x150 [97149.469016] ? set_kthread_struct+0x50/0x50 [97149.469760] ret_from_fork+0x1f/0x40 [97149.470433] CPU: 13 PID: 527323 Comm: ll_ost_io03_046 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.472655] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.474167] Call Trace: [97149.474702] dump_stack+0x41/0x60 [97149.475570] warn_alloc.cold.127+0x7b/0x108 [97149.476662] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.477797] __alloc_pages_nodemask+0x2e2/0x330 [97149.478935] kmalloc_order+0x28/0x90 [97149.479924] kmalloc_order_trace+0x1d/0xb0 [97149.480984] __kmalloc+0x203/0x250 [97149.481912] virtqueue_add+0x493/0xc70 [97149.482919] ? finish_wait+0x80/0x80 [97149.483909] virtqueue_add_sgs+0x80/0xa0 [97149.484941] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.486258] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.487415] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.488686] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.490059] scsi_queue_rq+0x512/0xb10 [97149.491059] __blk_mq_try_issue_directly+0x163/0x200 [97149.492294] blk_mq_request_issue_directly+0x4e/0xb0 [97149.493515] blk_mq_try_issue_list_directly+0x62/0x100 [97149.494767] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.495990] blk_mq_flush_plug_list+0x135/0x220 [97149.497171] blk_flush_plug_list+0xd7/0x100 [97149.498276] blk_finish_plug+0x25/0x36 [97149.499315] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.500788] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [97149.502118] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.503645] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.505137] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.506433] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.507654] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.508750] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.510065] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.511371] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.512690] ? newidle_balance+0x2b6/0x3b0 [97149.513741] ? internal_add_timer+0x42/0x70 [97149.514795] ? _cond_resched+0x15/0x30 [97149.515763] ? mutex_lock+0xe/0x30 [97149.516673] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.518048] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.519524] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.521024] ? finish_wait+0x80/0x80 [97149.521978] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.523198] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.524534] kthread+0x134/0x150 [97149.525288] ? set_kthread_struct+0x50/0x50 [97149.526300] ret_from_fork+0x1f/0x40 [97149.527248] CPU: 7 PID: 528150 Comm: ll_ost_io01_081 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.530275] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.532409] Call Trace: [97149.533221] dump_stack+0x41/0x60 [97149.534185] warn_alloc.cold.127+0x7b/0x108 [97149.535315] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.536485] ? __alloc_pages_nodemask+0x166/0x330 [97149.537696] __alloc_pages_nodemask+0x2e2/0x330 [97149.538879] kmalloc_order+0x28/0x90 [97149.539870] kmalloc_order_trace+0x1d/0xb0 [97149.540947] __kmalloc+0x203/0x250 [97149.541881] virtqueue_add+0x493/0xc70 [97149.542819] ? finish_wait+0x80/0x80 [97149.543778] virtqueue_add_sgs+0x80/0xa0 [97149.544813] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.546158] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.547292] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.548567] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.549964] scsi_queue_rq+0x512/0xb10 [97149.550971] __blk_mq_try_issue_directly+0x163/0x200 [97149.552207] blk_mq_request_issue_directly+0x4e/0xb0 [97149.553472] blk_mq_try_issue_list_directly+0x62/0x100 [97149.554752] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.555973] blk_mq_flush_plug_list+0x135/0x220 [97149.557136] blk_flush_plug_list+0xd7/0x100 [97149.558208] blk_finish_plug+0x25/0x36 [97149.559191] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.560638] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.561961] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.563517] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.565023] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.566340] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.567556] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.568680] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.570021] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.571308] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.572608] ? flush_work+0x42/0x1d0 [97149.573589] ? common_interrupt+0xa/0xf [97149.574622] ? _cond_resched+0x15/0x30 [97149.575607] ? mutex_lock+0xe/0x30 [97149.576525] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.577876] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.579365] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.580918] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.582175] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.583519] kthread+0x134/0x150 [97149.584409] ? set_kthread_struct+0x50/0x50 [97149.585484] ret_from_fork+0x1f/0x40 [97149.586441] CPU: 9 PID: 527871 Comm: ll_ost_io02_078 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97149.589187] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97149.590858] Call Trace: [97149.591460] dump_stack+0x41/0x60 [97149.592141] warn_alloc.cold.127+0x7b/0x108 [97149.592943] __alloc_pages_slowpath+0xcb2/0xcd0 [97149.593766] ? __alloc_pages_nodemask+0x166/0x330 [97149.594620] __alloc_pages_nodemask+0x2e2/0x330 [97149.595479] kmalloc_order+0x28/0x90 [97149.596363] kmalloc_order_trace+0x1d/0xb0 [97149.597454] __kmalloc+0x203/0x250 [97149.598391] virtqueue_add+0x493/0xc70 [97149.599418] ? finish_wait+0x80/0x80 [97149.600397] virtqueue_add_sgs+0x80/0xa0 [97149.601416] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97149.602748] ? scsi_alloc_sgtables+0x84/0x1c0 [97149.603858] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97149.605083] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97149.606436] scsi_queue_rq+0x512/0xb10 [97149.607425] __blk_mq_try_issue_directly+0x163/0x200 [97149.608663] blk_mq_request_issue_directly+0x4e/0xb0 [97149.609895] blk_mq_try_issue_list_directly+0x62/0x100 [97149.611145] blk_mq_sched_insert_requests+0xa4/0xf0 [97149.612370] blk_mq_flush_plug_list+0x135/0x220 [97149.613503] blk_flush_plug_list+0xd7/0x100 [97149.614574] blk_finish_plug+0x25/0x36 [97149.615562] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97149.617007] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97149.618356] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.619918] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97149.621093] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97149.622254] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97149.623492] ofd_commitrw+0x5b4/0xd20 [ofd] [97149.624610] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.625915] obd_commitrw+0x1b6/0x370 [ptlrpc] [97149.627203] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97149.628506] ? flush_work+0x42/0x1d0 [97149.629459] ? internal_add_timer+0x42/0x70 [97149.630550] ? _cond_resched+0x15/0x30 [97149.631535] ? mutex_lock+0xe/0x30 [97149.632461] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97149.633747] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97149.635213] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97149.636731] ? lu_context_exit+0x12/0x70 [obdclass] [97149.638021] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97149.639268] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97149.640669] kthread+0x134/0x150 [97149.641571] ? set_kthread_struct+0x50/0x50 [97149.642632] ret_from_fork+0x1f/0x40 [97156.419706] warn_alloc: 7 callbacks suppressed [97156.419710] ll_ost_io01_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97156.424008] CPU: 7 PID: 527029 Comm: ll_ost_io01_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97156.425618] ll_ost_io01_017: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97156.426976] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97156.426980] Call Trace: [97156.429546] / [97156.429969] ll_ost_io00_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97156.431755] dump_stack+0x41/0x60 [97156.432659] / [97156.432659] ,mems_allowed=0 [97156.433377] warn_alloc.cold.127+0x7b/0x108 [97156.435931] ,mems_allowed=0 [97156.436934] __alloc_pages_slowpath+0xcb2/0xcd0 [97156.438531] __alloc_pages_nodemask+0x2e2/0x330 [97156.439005] ll_ost_io02_028: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97156.444591] ll_ost_io03_008: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97156.446057] kmalloc_order+0x28/0x90 [97156.449936] kmalloc_order_trace+0x1d/0xb0 [97156.451063] __kmalloc+0x203/0x250 [97156.452082] ? __wake_up_common+0x7a/0x190 [97156.453202] virtqueue_add+0x493/0xc70 [97156.454233] ? finish_wait+0x80/0x80 [97156.455198] virtqueue_add_sgs+0x80/0xa0 [97156.456233] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97156.457592] ? scsi_alloc_sgtables+0x84/0x1c0 [97156.458758] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97156.460039] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97156.461477] scsi_queue_rq+0x512/0xb10 [97156.462508] __blk_mq_try_issue_directly+0x163/0x200 [97156.463754] blk_mq_request_issue_directly+0x4e/0xb0 [97156.464982] blk_mq_try_issue_list_directly+0x62/0x100 [97156.466249] blk_mq_sched_insert_requests+0xa4/0xf0 [97156.467462] blk_mq_flush_plug_list+0x135/0x220 [97156.468677] blk_flush_plug_list+0xd7/0x100 [97156.469779] blk_finish_plug+0x25/0x36 [97156.470773] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97156.472205] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97156.473541] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.475084] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.476557] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97156.477854] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97156.479070] ofd_commitrw+0x5b4/0xd20 [ofd] [97156.480199] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.481561] obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.482885] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97156.484192] ? flush_work+0x42/0x1d0 [97156.485161] ? internal_add_timer+0x42/0x70 [97156.486238] ? _cond_resched+0x15/0x30 [97156.487216] ? mutex_lock+0xe/0x30 [97156.488128] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97156.489490] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97156.490708] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97156.491802] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97156.492689] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97156.493661] kthread+0x134/0x150 [97156.494309] ? set_kthread_struct+0x50/0x50 [97156.495060] ret_from_fork+0x1f/0x40 [97156.495715] warn_alloc_show_mem: 9 callbacks suppressed [97156.495717] Mem-Info: [97156.495716] CPU: 14 PID: 527023 Comm: ll_ost_io03_008 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97156.496611] active_anon:27936 inactive_anon:199651 isolated_anon:0 active_file:31215841 inactive_file:3588271 isolated_file:30 unevictable:0 dirty:1481 writeback:0 slab_reclaimable:1001085 slab_unreclaimable:606119 mapped:22079 shmem:146 pagetables:5693 bounce:0 free:1375228 free_pcp:465 free_cma:0 [97156.497383] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97156.499916] Node 0 active_anon:111744kB inactive_anon:798604kB active_file:124810664kB inactive_file:14353032kB unevictable:0kB isolated(anon):0kB isolated(file):120kB mapped:88316kB dirty:5924kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35472kB pagetables:22772kB all_unreclaimable? no [97156.507794] Call Trace: [97156.509290] Node 0 [97156.516034] dump_stack+0x41/0x60 [97156.516717] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97156.517462] warn_alloc.cold.127+0x7b/0x108 [97156.518429] lowmem_reserve[]: [97156.523365] __alloc_pages_slowpath+0xcb2/0xcd0 [97156.524476] 0 [97156.525391] ? __alloc_pages_nodemask+0x166/0x330 [97156.526577] 913 [97156.527262] __alloc_pages_nodemask+0x2e2/0x330 [97156.528469] 150007 [97156.529189] kmalloc_order+0x28/0x90 [97156.530352] 150007 [97156.531123] kmalloc_order_trace+0x1d/0xb0 [97156.532124] 150007 [97156.532879] __kmalloc+0x203/0x250 [97156.534697] virtqueue_add+0x493/0xc70 [97156.535657] Node 0 [97156.536277] ? finish_wait+0x80/0x80 [97156.537336] DMA32 free:596660kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:12kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97156.538105] virtqueue_add_sgs+0x80/0xa0 [97156.539089] lowmem_reserve[]: 0 [97156.544187] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97156.545242] 0 [97156.546236] ? scsi_alloc_sgtables+0x84/0x1c0 [97156.547592] 149093 [97156.548318] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97156.549522] 149093 [97156.550288] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97156.551610] 149093 [97156.552414] scsi_queue_rq+0x512/0xb10 [97156.554638] __blk_mq_try_issue_directly+0x163/0x200 [97156.555483] Node 0 [97156.556181] blk_mq_request_issue_directly+0x4e/0xb0 [97156.557282] Normal free:5467704kB min:49268kB low:201940kB high:354612kB active_anon:111364kB inactive_anon:798600kB active_file:124290976kB inactive_file:14353020kB unevictable:0kB writepending:5924kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1576kB local_pcp:8kB free_cma:0kB [97156.558139] blk_mq_try_issue_list_directly+0x62/0x100 [97156.559248] lowmem_reserve[]: 0 [97156.565106] blk_mq_sched_insert_requests+0xa4/0xf0 [97156.566394] 0 [97156.567389] blk_mq_flush_plug_list+0x135/0x220 [97156.568611] 0 [97156.569315] blk_flush_plug_list+0xd7/0x100 [97156.570255] 0 [97156.571013] blk_finish_plug+0x25/0x36 [97156.571853] 0 [97156.572591] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97156.574369] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97156.575736] Node 0 [97156.576428] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.577636] DMA: [97156.578442] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.579959] 0*4kB [97156.580730] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97156.582244] 0*8kB [97156.583034] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97156.584077] 0*16kB [97156.584829] ofd_commitrw+0x5b4/0xd20 [ofd] [97156.585788] 0*32kB [97156.586593] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.587456] 0*64kB [97156.588273] obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.589160] 0*128kB [97156.589987] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97156.590855] 0*256kB [97156.591686] ? newidle_balance+0x2b6/0x3b0 [97156.592616] 0*512kB [97156.593467] ? internal_add_timer+0x42/0x70 [97156.594298] 1*1024kB [97156.595120] ? _cond_resched+0x15/0x30 [97156.595948] (U) [97156.596751] ? mutex_lock+0xe/0x30 [97156.597536] 1*2048kB [97156.598262] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97156.598953] (M) [97156.599769] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97156.600705] 2*4096kB [97156.601443] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97156.602467] (M) [97156.603310] ? finish_wait+0x80/0x80 [97156.604384] = 11264kB [97156.605160] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97156.605872] Node 0 [97156.606703] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97156.607541] DMA32: [97156.608307] kthread+0x134/0x150 [97156.609236] 21*4kB [97156.609999] ? set_kthread_struct+0x50/0x50 [97156.610674] (UM) [97156.611437] ret_from_fork+0x1f/0x40 [97156.612271] 24*8kB [97156.612829] CPU: 9 PID: 527185 Comm: ll_ost_io02_028 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97156.613641] (UM) [97156.614269] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97156.616476] 26*16kB [97156.617088] Call Trace: [97156.618654] (UME) [97156.619433] dump_stack+0x41/0x60 [97156.620019] 44*32kB [97156.620589] warn_alloc.cold.127+0x7b/0x108 [97156.621275] (UM) [97156.621877] __alloc_pages_slowpath+0xcb2/0xcd0 [97156.622698] 46*64kB [97156.623273] ? __alloc_pages_nodemask+0x166/0x330 [97156.624145] (UM) [97156.624700] __alloc_pages_nodemask+0x2e2/0x330 [97156.625600] 46*128kB [97156.626151] kmalloc_order+0x28/0x90 [97156.627015] (UMH) [97156.627588] kmalloc_order_trace+0x1d/0xb0 [97156.628342] 45*256kB [97156.628874] __kmalloc+0x203/0x250 [97156.629683] (UMH) [97156.630257] virtqueue_add+0x493/0xc70 [97156.630976] 48*512kB [97156.631510] ? finish_wait+0x80/0x80 [97156.632260] (UMEH) [97156.632831] virtqueue_add_sgs+0x80/0xa0 [97156.633564] 39*1024kB [97156.634139] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97156.634929] (UM) [97156.635518] ? scsi_alloc_sgtables+0x84/0x1c0 [97156.636533] 43*2048kB [97156.637099] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97156.638028] (UM) [97156.638623] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97156.639574] 103*4096kB [97156.640106] scsi_queue_rq+0x512/0xb10 [97156.641149] (UM) [97156.641730] __blk_mq_try_issue_directly+0x163/0x200 [97156.642498] = 596916kB [97156.643033] blk_mq_request_issue_directly+0x4e/0xb0 [97156.643959] Node 0 [97156.644536] blk_mq_try_issue_list_directly+0x62/0x100 [97156.645519] Normal: [97156.646082] blk_mq_sched_insert_requests+0xa4/0xf0 [97156.647042] 725172*4kB [97156.647618] blk_mq_flush_plug_list+0x135/0x220 [97156.648543] (UME) [97156.649148] blk_flush_plug_list+0xd7/0x100 [97156.650084] 284501*8kB [97156.650621] blk_finish_plug+0x25/0x36 [97156.651466] (UME) [97156.652100] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97156.652834] 17613*16kB [97156.653395] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97156.654411] (UMEH) [97156.655002] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.655951] 24*32kB [97156.656485] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.657759] (UH) [97156.658319] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97156.659439] 0*64kB [97156.659958] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97156.660920] 0*128kB [97156.661467] ofd_commitrw+0x5b4/0xd20 [ofd] [97156.662361] 0*256kB [97156.662924] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.663707] 0*512kB [97156.664269] obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.665188] 0*1024kB [97156.665737] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97156.666539] 0*2048kB [97156.667123] ? flush_work+0x42/0x1d0 [97156.668000] 0*4096kB [97156.668567] ? internal_add_timer+0x42/0x70 [97156.669291] = 5459272kB [97156.669844] ? _cond_resched+0x15/0x30 [97156.670659] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97156.671240] ? mutex_lock+0xe/0x30 [97156.671998] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97156.673463] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97156.674158] 34666832 total pagecache pages [97156.675607] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97156.676513] 1539 pages in swap cache [97156.677356] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97156.678355] Swap cache stats: add 1604095, delete 1602579, find 82812/99801 [97156.679103] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97156.680176] Free swap = 5340268kB [97156.681406] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97156.682229] Total swap = 11075580kB [97156.682980] kthread+0x134/0x150 [97156.683919] 39321462 pages RAM [97156.684652] ? set_kthread_struct+0x50/0x50 [97156.685329] 0 pages HighMem/MovableOnly [97156.686014] ret_from_fork+0x1f/0x40 [97156.686837] 905848 pages reserved [97156.687626] CPU: 4 PID: 527073 Comm: ll_ost_io01_017 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97156.688313] 0 pages hwpoisoned [97156.689287] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97156.694368] Call Trace: [97156.695138] dump_stack+0x41/0x60 [97156.695815] warn_alloc.cold.127+0x7b/0x108 [97156.696608] __alloc_pages_slowpath+0xcb2/0xcd0 [97156.697444] ? __alloc_pages_nodemask+0x166/0x330 [97156.698317] __alloc_pages_nodemask+0x2e2/0x330 [97156.699278] kmalloc_order+0x28/0x90 [97156.700038] kmalloc_order_trace+0x1d/0xb0 [97156.700856] __kmalloc+0x203/0x250 [97156.701702] virtqueue_add+0x493/0xc70 [97156.702713] virtqueue_add_sgs+0x80/0xa0 [97156.703757] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97156.705078] ? scsi_alloc_sgtables+0x84/0x1c0 [97156.706194] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97156.707337] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97156.708425] scsi_queue_rq+0x512/0xb10 [97156.709179] __blk_mq_try_issue_directly+0x163/0x200 [97156.710093] blk_mq_request_issue_directly+0x4e/0xb0 [97156.711514] blk_mq_try_issue_list_directly+0x62/0x100 [97156.712612] blk_mq_sched_insert_requests+0xa4/0xf0 [97156.713746] blk_mq_flush_plug_list+0x135/0x220 [97156.714901] blk_flush_plug_list+0xd7/0x100 [97156.715894] blk_finish_plug+0x25/0x36 [97156.716725] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97156.717861] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97156.719135] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.720576] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.721769] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97156.722923] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97156.723883] ofd_commitrw+0x5b4/0xd20 [ofd] [97156.724702] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.725692] obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.726634] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97156.727628] ? flush_work+0x42/0x1d0 [97156.728581] ? internal_add_timer+0x42/0x70 [97156.729523] ? _cond_resched+0x15/0x30 [97156.730287] ? mutex_lock+0xe/0x30 [97156.731215] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97156.732526] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97156.734012] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97156.735356] ? lu_context_exit+0x54/0x70 [obdclass] [97156.736425] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97156.737592] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97156.738619] kthread+0x134/0x150 [97156.739456] ? set_kthread_struct+0x50/0x50 [97156.740572] ret_from_fork+0x1f/0x40 [97156.741575] CPU: 2 PID: 527174 Comm: ll_ost_io00_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97156.744551] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97156.746395] Call Trace: [97156.747117] dump_stack+0x41/0x60 [97156.747999] warn_alloc.cold.127+0x7b/0x108 [97156.749052] __alloc_pages_slowpath+0xcb2/0xcd0 [97156.749923] ? __alloc_pages_nodemask+0x166/0x330 [97156.750892] __alloc_pages_nodemask+0x2e2/0x330 [97156.751760] kmalloc_order+0x28/0x90 [97156.752448] kmalloc_order_trace+0x1d/0xb0 [97156.753264] __kmalloc+0x203/0x250 [97156.754008] ? __wake_up_common+0x7a/0x190 [97156.754759] virtqueue_add+0x493/0xc70 [97156.755468] virtqueue_add_sgs+0x80/0xa0 [97156.756202] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97156.757147] ? scsi_alloc_sgtables+0x84/0x1c0 [97156.758102] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97156.759295] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97156.760372] scsi_queue_rq+0x512/0xb10 [97156.761313] __blk_mq_try_issue_directly+0x163/0x200 [97156.762533] blk_mq_request_issue_directly+0x4e/0xb0 [97156.763650] blk_mq_try_issue_list_directly+0x62/0x100 [97156.764794] blk_mq_sched_insert_requests+0xa4/0xf0 [97156.765796] blk_mq_flush_plug_list+0x135/0x220 [97156.766677] blk_flush_plug_list+0xd7/0x100 [97156.767641] blk_finish_plug+0x25/0x36 [97156.768556] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97156.769784] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97156.770951] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.772484] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97156.773992] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97156.775152] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97156.776290] ofd_commitrw+0x5b4/0xd20 [ofd] [97156.777260] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.778592] obd_commitrw+0x1b6/0x370 [ptlrpc] [97156.779859] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97156.781159] ? flush_work+0x42/0x1d0 [97156.782102] ? internal_add_timer+0x42/0x70 [97156.783186] ? _cond_resched+0x15/0x30 [97156.784187] ? mutex_lock+0xe/0x30 [97156.785100] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97156.786452] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97156.787932] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97156.789433] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97156.790683] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97156.791806] kthread+0x134/0x150 [97156.792633] ? set_kthread_struct+0x50/0x50 [97156.793519] ret_from_fork+0x1f/0x40 [97160.016618] ll_ost_io02_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97160.017094] ll_ost_io03_078: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97160.017100] ll_ost_io03_005: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97160.017111] CPU: 13 PID: 527019 Comm: ll_ost_io03_005 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97160.017115] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97160.017116] Call Trace: [97160.017120] ll_ost_io01_000: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [97160.017121] dump_stack+0x41/0x60 [97160.017128] ,mems_allowed=0 [97160.017130] warn_alloc.cold.127+0x7b/0x108 [97160.017132] ll_ost_io04_004: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97160.017137] __alloc_pages_slowpath+0xcb2/0xcd0 [97160.017139] /,mems_allowed=0 [97160.017141] ? __alloc_pages_nodemask+0x166/0x330 [97160.017145] __alloc_pages_nodemask+0x2e2/0x330 [97160.017148] kmalloc_order+0x28/0x90 [97160.017154] kmalloc_order_trace+0x1d/0xb0 [97160.017158] __kmalloc+0x203/0x250 [97160.017163] virtqueue_add+0x493/0xc70 [97160.017170] ? finish_wait+0x80/0x80 [97160.017175] virtqueue_add_sgs+0x80/0xa0 [97160.017180] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97160.017186] ? scsi_alloc_sgtables+0x84/0x1c0 [97160.017192] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97160.017195] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97160.017199] scsi_queue_rq+0x512/0xb10 [97160.017202] __blk_mq_try_issue_directly+0x163/0x200 [97160.017208] blk_mq_request_issue_directly+0x4e/0xb0 [97160.017213] blk_mq_try_issue_list_directly+0x62/0x100 [97160.017216] blk_mq_sched_insert_requests+0xa4/0xf0 [97160.017220] blk_mq_flush_plug_list+0x135/0x220 [97160.017225] blk_flush_plug_list+0xd7/0x100 [97160.017231] blk_finish_plug+0x25/0x36 [97160.017236] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97160.017275] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97160.017313] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.017342] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.017370] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97160.017398] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97160.017418] ofd_commitrw+0x5b4/0xd20 [ofd] [97160.017429] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.017521] obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.017590] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97160.017662] ? set_next_entity+0xb5/0x1e0 [97160.017667] ? internal_add_timer+0x42/0x70 [97160.017670] ? _cond_resched+0x15/0x30 [97160.017673] ? mutex_lock+0xe/0x30 [97160.017675] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97160.017746] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97160.017812] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97160.017874] ? finish_wait+0x80/0x80 [97160.017876] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97160.017945] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97160.018013] kthread+0x134/0x150 [97160.018017] ? set_kthread_struct+0x50/0x50 [97160.018019] ret_from_fork+0x1f/0x40 [97160.018024] warn_alloc_show_mem: 4 callbacks suppressed [97160.018026] Mem-Info: [97160.018025] CPU: 16 PID: 527163 Comm: ll_ost_io04_004 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97160.018028] active_anon:27936 inactive_anon:199840 isolated_anon:0 active_file:31803889 inactive_file:3506245 isolated_file:30 unevictable:0 dirty:599 writeback:0 slab_reclaimable:1001080 slab_unreclaimable:606664 mapped:22207 shmem:146 pagetables:5693 bounce:0 free:868233 free_pcp:449 free_cma:0 [97160.018030] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97160.018032] Call Trace: [97160.018032] Node 0 active_anon:111744kB inactive_anon:799360kB active_file:127215556kB inactive_file:14024980kB unevictable:0kB isolated(anon):0kB isolated(file):120kB mapped:88828kB dirty:2396kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35472kB pagetables:22772kB all_unreclaimable? no [97160.018038] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97160.018035] dump_stack+0x41/0x60 [97160.018043] lowmem_reserve[]: 0 913 150007 150007 150007 [97160.018043] warn_alloc.cold.127+0x7b/0x108 [97160.018046] Node 0 DMA32 free:596500kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:12kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97160.018049] __alloc_pages_slowpath+0xcb2/0xcd0 [97160.018051] lowmem_reserve[]: 0 0 149093 149093 149093 [97160.018053] ? __alloc_pages_nodemask+0x166/0x330 [97160.018055] Node 0 Normal free:2865168kB min:49268kB low:201940kB high:354612kB active_anon:111364kB inactive_anon:799356kB active_file:127215696kB inactive_file:14025224kB unevictable:0kB writepending:2396kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1796kB local_pcp:116kB free_cma:0kB [97160.018057] __alloc_pages_nodemask+0x2e2/0x330 [97160.018060] lowmem_reserve[]: 0 0 0 0 0 [97160.018063] Node 0 DMA: [97160.018060] kmalloc_order+0x28/0x90 [97160.018065] 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB [97160.018067] kmalloc_order_trace+0x1d/0xb0 [97160.018069] 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB [97160.018071] __kmalloc+0x203/0x250 [97160.018074] (M) = 11264kB [97160.018075] Node 0 DMA32: 21*4kB (UM) 24*8kB (UMH) [97160.018076] virtqueue_add+0x493/0xc70 [97160.018079] 26*16kB (UME) 37*32kB (UMH) 45*64kB (UMH) 47*128kB (UM) 46*256kB (UMH) [97160.018083] ? finish_wait+0x80/0x80 [97160.018085] 47*512kB (UMEH) 39*1024kB (UM) 43*2048kB (UM) [97160.018087] virtqueue_add_sgs+0x80/0xa0 [97160.018090] 103*4096kB (UM) = 596500kB [97160.018092] Node 0 Normal: 40695*4kB (UME) 299684*8kB (UME) 19072*16kB [97160.018093] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97160.018096] (UMEH) 1*32kB (U) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB [97160.018099] ? scsi_alloc_sgtables+0x84/0x1c0 [97160.018102] = 2865436kB [97160.018104] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97160.018106] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97160.018105] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97160.018107] 35310013 total pagecache pages [97160.018108] 1541 pages in swap cache [97160.018109] Swap cache stats: add 1604134, delete 1602616, find 82822/99814 [97160.018110] Free swap = 5340268kB [97160.018111] Total swap = 11075580kB [97160.018109] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97160.018111] 39321462 pages RAM [97160.018112] 0 pages HighMem/MovableOnly [97160.018113] 905848 pages reserved [97160.018113] 0 pages hwpoisoned [97160.018114] scsi_queue_rq+0x512/0xb10 [97160.018117] __blk_mq_try_issue_directly+0x163/0x200 [97160.018124] blk_mq_request_issue_directly+0x4e/0xb0 [97160.018128] blk_mq_try_issue_list_directly+0x62/0x100 [97160.018131] blk_mq_sched_insert_requests+0xa4/0xf0 [97160.018135] blk_mq_flush_plug_list+0x135/0x220 [97160.018139] blk_flush_plug_list+0xd7/0x100 [97160.018143] blk_finish_plug+0x25/0x36 [97160.018148] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97160.018186] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97160.018217] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.018244] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.018272] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97160.018302] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97160.018325] ofd_commitrw+0x5b4/0xd20 [ofd] [97160.018344] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.018473] obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.018598] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97160.018712] ? flush_work+0x42/0x1d0 [97160.018717] ? internal_add_timer+0x42/0x70 [97160.018721] ? _cond_resched+0x15/0x30 [97160.018724] ? mutex_lock+0xe/0x30 [97160.018728] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97160.018847] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97160.018959] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97160.019089] ? lprocfs_counter_add+0x89/0x140 [obdclass] [97160.019168] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97160.019288] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97160.019406] kthread+0x134/0x150 [97160.019411] ? set_kthread_struct+0x50/0x50 [97160.019414] ret_from_fork+0x1f/0x40 [97160.019421] CPU: 7 PID: 188100 Comm: ll_ost_io01_000 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97160.019426] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97160.019427] Call Trace: [97160.019430] dump_stack+0x41/0x60 [97160.019436] warn_alloc.cold.127+0x7b/0x108 [97160.019442] __alloc_pages_slowpath+0xcb2/0xcd0 [97160.019446] ? __alloc_pages_nodemask+0x166/0x330 [97160.019449] __alloc_pages_nodemask+0x2e2/0x330 [97160.019453] kmalloc_order+0x28/0x90 [97160.019457] kmalloc_order_trace+0x1d/0xb0 [97160.019461] __kmalloc+0x203/0x250 [97160.019467] virtqueue_add+0x493/0xc70 [97160.019472] ? finish_wait+0x80/0x80 [97160.019476] virtqueue_add_sgs+0x80/0xa0 [97160.019481] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97160.019487] ? scsi_alloc_sgtables+0x84/0x1c0 [97160.019493] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97160.019496] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97160.019500] scsi_queue_rq+0x512/0xb10 [97160.019503] __blk_mq_try_issue_directly+0x163/0x200 [97160.019509] blk_mq_request_issue_directly+0x4e/0xb0 [97160.019513] blk_mq_try_issue_list_directly+0x62/0x100 [97160.019518] blk_mq_sched_insert_requests+0xa4/0xf0 [97160.019522] blk_mq_flush_plug_list+0x135/0x220 [97160.019526] blk_flush_plug_list+0xd7/0x100 [97160.019531] blk_finish_plug+0x25/0x36 [97160.019536] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97160.019573] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97160.019605] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.019632] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.019675] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97160.019705] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97160.019727] ofd_commitrw+0x5b4/0xd20 [ofd] [97160.019746] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.019890] obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.020016] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97160.020133] ? flush_work+0x42/0x1d0 [97160.020138] ? internal_add_timer+0x42/0x70 [97160.020141] ? _cond_resched+0x15/0x30 [97160.020144] ? mutex_lock+0xe/0x30 [97160.020147] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97160.020265] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97160.020380] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97160.020503] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97160.023383] ,cpuset= [97160.026258] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97160.029355] / [97160.031643] kthread+0x134/0x150 [97160.032549] ,mems_allowed=0 [97160.035073] ? set_kthread_struct+0x50/0x50 [97160.037113] ret_from_fork+0x1f/0x40 [97160.267604] CPU: 14 PID: 527551 Comm: ll_ost_io03_078 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97160.269683] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97160.271171] Call Trace: [97160.271686] dump_stack+0x41/0x60 [97160.272327] warn_alloc.cold.127+0x7b/0x108 [97160.273161] __alloc_pages_slowpath+0xcb2/0xcd0 [97160.274235] ? __alloc_pages_nodemask+0x166/0x330 [97160.275412] __alloc_pages_nodemask+0x2e2/0x330 [97160.276547] kmalloc_order+0x28/0x90 [97160.277375] kmalloc_order_trace+0x1d/0xb0 [97160.278230] __kmalloc+0x203/0x250 [97160.279074] virtqueue_add+0x493/0xc70 [97160.279810] ? finish_wait+0x80/0x80 [97160.280494] virtqueue_add_sgs+0x80/0xa0 [97160.281222] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97160.282370] ? scsi_alloc_sgtables+0x84/0x1c0 [97160.283460] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97160.284424] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97160.285402] scsi_queue_rq+0x512/0xb10 [97160.286103] __blk_mq_try_issue_directly+0x163/0x200 [97160.287003] blk_mq_request_issue_directly+0x4e/0xb0 [97160.287890] blk_mq_try_issue_list_directly+0x62/0x100 [97160.288791] blk_mq_sched_insert_requests+0xa4/0xf0 [97160.289668] blk_mq_flush_plug_list+0x135/0x220 [97160.290500] blk_flush_plug_list+0xd7/0x100 [97160.291265] blk_finish_plug+0x25/0x36 [97160.291953] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97160.292982] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97160.293922] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.295033] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.296203] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97160.297206] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97160.298330] ofd_commitrw+0x5b4/0xd20 [ofd] [97160.299193] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.300276] obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.301268] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97160.302455] ? set_next_entity+0xb5/0x1e0 [97160.303241] ? internal_add_timer+0x42/0x70 [97160.304065] ? _cond_resched+0x15/0x30 [97160.304901] ? mutex_lock+0xe/0x30 [97160.305588] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97160.306608] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97160.307696] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97160.308944] ? finish_wait+0x80/0x80 [97160.309901] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97160.311158] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97160.312521] kthread+0x134/0x150 [97160.313432] ? set_kthread_struct+0x50/0x50 [97160.314507] ret_from_fork+0x1f/0x40 [97160.315476] CPU: 9 PID: 527071 Comm: ll_ost_io02_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97160.318161] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97160.320303] Call Trace: [97160.321067] dump_stack+0x41/0x60 [97160.321981] warn_alloc.cold.127+0x7b/0x108 [97160.322824] __alloc_pages_slowpath+0xcb2/0xcd0 [97160.323706] ? __alloc_pages_nodemask+0x166/0x330 [97160.324616] __alloc_pages_nodemask+0x2e2/0x330 [97160.325516] kmalloc_order+0x28/0x90 [97160.326287] kmalloc_order_trace+0x1d/0xb0 [97160.327169] __kmalloc+0x203/0x250 [97160.327873] virtqueue_add+0x493/0xc70 [97160.328695] ? finish_wait+0x80/0x80 [97160.329520] virtqueue_add_sgs+0x80/0xa0 [97160.330356] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97160.331653] ? scsi_alloc_sgtables+0x84/0x1c0 [97160.332754] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97160.333793] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97160.334971] scsi_queue_rq+0x512/0xb10 [97160.335760] __blk_mq_try_issue_directly+0x163/0x200 [97160.336682] blk_mq_request_issue_directly+0x4e/0xb0 [97160.337583] blk_mq_try_issue_list_directly+0x62/0x100 [97160.338594] blk_mq_sched_insert_requests+0xa4/0xf0 [97160.339682] blk_mq_flush_plug_list+0x135/0x220 [97160.340733] blk_flush_plug_list+0xd7/0x100 [97160.341643] blk_finish_plug+0x25/0x36 [97160.342499] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97160.343770] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97160.344868] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.346383] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97160.347932] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97160.349123] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97160.350112] ofd_commitrw+0x5b4/0xd20 [ofd] [97160.351095] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.352235] obd_commitrw+0x1b6/0x370 [ptlrpc] [97160.353389] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97160.354523] ? flush_work+0x42/0x1d0 [97160.355314] ? internal_add_timer+0x42/0x70 [97160.356327] ? _cond_resched+0x15/0x30 [97160.357169] ? mutex_lock+0xe/0x30 [97160.357914] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97160.358944] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97160.360293] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97160.361490] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97160.362651] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97160.363701] kthread+0x134/0x150 [97160.364367] ? set_kthread_struct+0x50/0x50 [97160.365167] ret_from_fork+0x1f/0x40 [97162.396765] warn_alloc: 17 callbacks suppressed [97162.396770] ll_ost_io01_033: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97162.396774] ll_ost_io04_035: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97162.398164] ,cpuset= [97162.400779] ,cpuset= [97162.403291] / [97162.404964] / [97162.404964] ,mems_allowed=0 [97162.405664] ,mems_allowed=0 [97162.406371] CPU: 5 PID: 527137 Comm: ll_ost_io01_033 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97162.408184] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97162.413874] Call Trace: [97162.414670] dump_stack+0x41/0x60 [97162.415629] warn_alloc.cold.127+0x7b/0x108 [97162.416740] __alloc_pages_slowpath+0xcb2/0xcd0 [97162.417931] ? __alloc_pages_nodemask+0x166/0x330 [97162.419152] __alloc_pages_nodemask+0x2e2/0x330 [97162.420323] kmalloc_order+0x28/0x90 [97162.421307] kmalloc_order_trace+0x1d/0xb0 [97162.422398] __kmalloc+0x203/0x250 [97162.423361] virtqueue_add+0x493/0xc70 [97162.424388] virtqueue_add_sgs+0x80/0xa0 [97162.425449] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97162.426817] ? scsi_alloc_sgtables+0x84/0x1c0 [97162.427942] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97162.429210] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97162.430617] scsi_queue_rq+0x512/0xb10 [97162.431643] __blk_mq_try_issue_directly+0x163/0x200 [97162.432894] blk_mq_request_issue_directly+0x4e/0xb0 [97162.434140] blk_mq_try_issue_list_directly+0x62/0x100 [97162.435437] blk_mq_sched_insert_requests+0xa4/0xf0 [97162.436691] blk_mq_flush_plug_list+0x135/0x220 [97162.437882] blk_flush_plug_list+0xd7/0x100 [97162.439000] blk_finish_plug+0x25/0x36 [97162.440064] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97162.441532] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97162.442863] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97162.444426] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97162.445942] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97162.447262] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97162.448523] ofd_commitrw+0x5b4/0xd20 [ofd] [97162.449694] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97162.451027] obd_commitrw+0x1b6/0x370 [ptlrpc] [97162.452324] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97162.453659] ? flush_work+0x42/0x1d0 [97162.454669] ? internal_add_timer+0x42/0x70 [97162.455749] ? _cond_resched+0x15/0x30 [97162.456790] ? mutex_lock+0xe/0x30 [97162.457726] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97162.459124] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97162.460657] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97162.462229] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97162.463477] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97162.464833] kthread+0x134/0x150 [97162.465726] ? set_kthread_struct+0x50/0x50 [97162.466808] ret_from_fork+0x1f/0x40 [97162.467799] warn_alloc_show_mem: 4 callbacks suppressed [97162.467802] Mem-Info: [97162.467800] CPU: 19 PID: 527352 Comm: ll_ost_io04_035 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97162.469110] active_anon:27936 inactive_anon:199783 isolated_anon:0 active_file:31449525 inactive_file:3417860 isolated_file:21 unevictable:0 dirty:1335 writeback:1 slab_reclaimable:1001080 slab_unreclaimable:606294 mapped:22310 shmem:146 pagetables:5693 bounce:0 free:1310110 free_pcp:434 free_cma:0 [97162.469962] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97162.472839] Node 0 active_anon:111744kB inactive_anon:799132kB active_file:125806552kB inactive_file:13671440kB unevictable:0kB isolated(anon):0kB isolated(file):84kB mapped:89240kB dirty:5340kB writeback:4kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35472kB pagetables:22772kB all_unreclaimable? no [97162.480788] Call Trace: [97162.482835] Node 0 [97162.489588] dump_stack+0x41/0x60 [97162.490412] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97162.491171] warn_alloc.cold.127+0x7b/0x108 [97162.492176] lowmem_reserve[]: [97162.497108] __alloc_pages_slowpath+0xcb2/0xcd0 [97162.498252] 0 [97162.499197] ? __alloc_pages_nodemask+0x166/0x330 [97162.500434] 913 [97162.501165] __alloc_pages_nodemask+0x2e2/0x330 [97162.502458] 150007 [97162.503232] kmalloc_order+0x28/0x90 [97162.504489] 150007 [97162.505306] kmalloc_order_trace+0x1d/0xb0 [97162.506400] 150007 [97162.507210] __kmalloc+0x203/0x250 [97162.509206] virtqueue_add+0x493/0xc70 [97162.510251] Node 0 [97162.510937] ? finish_wait+0x80/0x80 [97162.511982] DMA32 free:596692kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:12kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97162.512769] virtqueue_add_sgs+0x80/0xa0 [97162.513857] lowmem_reserve[]: [97162.519207] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97162.520355] 0 [97162.521341] ? scsi_alloc_sgtables+0x84/0x1c0 [97162.522801] 0 [97162.523561] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97162.524828] 149093 [97162.525598] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97162.527003] 149093 [97162.527832] scsi_queue_rq+0x512/0xb10 [97162.529386] 149093 [97162.530196] __blk_mq_try_issue_directly+0x163/0x200 [97162.532171] blk_mq_request_issue_directly+0x4e/0xb0 [97162.533532] Node 0 [97162.534268] blk_mq_try_issue_list_directly+0x62/0x100 [97162.535627] Normal free:4539232kB min:49268kB low:201940kB high:354612kB active_anon:111364kB inactive_anon:799132kB active_file:125892308kB inactive_file:13670996kB unevictable:0kB writepending:5124kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1604kB local_pcp:0kB free_cma:0kB [97162.536434] blk_mq_sched_insert_requests+0xa4/0xf0 [97162.537815] lowmem_reserve[]: [97162.543775] blk_mq_flush_plug_list+0x135/0x220 [97162.545165] 0 [97162.546168] blk_flush_plug_list+0xd7/0x100 [97162.547480] 0 [97162.548230] blk_finish_plug+0x25/0x36 [97162.549415] 0 [97162.550131] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97162.551224] 0 [97162.551933] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [97162.553413] 0 [97162.554154] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97162.556275] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97162.557927] Node 0 [97162.558563] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97162.560111] DMA: [97162.560984] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97162.562358] 0*4kB [97162.563157] ofd_commitrw+0x5b4/0xd20 [ofd] [97162.564474] 0*8kB [97162.565261] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97162.566418] 0*16kB [97162.567229] obd_commitrw+0x1b6/0x370 [ptlrpc] [97162.568505] 0*32kB [97162.569337] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97162.570585] 0*64kB [97162.571448] ? flush_work+0x42/0x1d0 [97162.572750] 0*128kB [97162.573594] ? internal_add_timer+0x42/0x70 [97162.574673] 0*256kB [97162.575480] ? _cond_resched+0x15/0x30 [97162.576694] 0*512kB [97162.577487] ? mutex_lock+0xe/0x30 [97162.578612] 1*1024kB [97162.579401] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97162.580430] (U) [97162.581279] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97162.582616] 1*2048kB [97162.583347] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97162.584803] (M) [97162.585641] ? finish_wait+0x80/0x80 [97162.587145] 2*4096kB [97162.587897] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97162.588916] (M) [97162.589776] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97162.590960] = 11264kB [97162.591681] kthread+0x134/0x150 [97162.593028] Node 0 [97162.593868] ? set_kthread_struct+0x50/0x50 [97162.594841] DMA32: [97162.595630] ret_from_fork+0x1f/0x40 [97162.596800] 21*4kB (UM) 24*8kB (UMH) 26*16kB (UME) 43*32kB (UMH) 48*64kB (UMH) 48*128kB (UMH) 45*256kB (UM) 47*512kB (UMEH) 39*1024kB (UM) 43*2048kB (UM) 103*4096kB (UM) = 596756kB [97162.602632] Node 0 Normal: 274117*4kB (UME) 382189*8kB (UME) 21005*16kB (UMEH) 4*32kB (UH) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 4490188kB [97162.606175] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97162.608244] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97162.610234] 34904702 total pagecache pages [97162.611385] 1541 pages in swap cache [97162.612429] Swap cache stats: add 1604158, delete 1602640, find 82822/99814 [97162.614176] Free swap = 5340012kB [97162.615188] Total swap = 11075580kB [97162.616212] 39321462 pages RAM [97162.617187] 0 pages HighMem/MovableOnly [97162.618283] 905848 pages reserved [97162.619266] 0 pages hwpoisoned [97259.180408] ll_ost_io02_039: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97259.183004] ll_ost_io02_074: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [97259.183352] CPU: 10 PID: 527244 Comm: ll_ost_io02_039 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97259.183355] ,mems_allowed=0 [97259.183357] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97259.183359] Call Trace: [97259.188245] dump_stack+0x41/0x60 [97259.189862] ll_ost_io04_036: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97259.191044] warn_alloc.cold.127+0x7b/0x108 [97259.192054] ,cpuset= [97259.192479] ll_ost_io01_060: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97259.192720] __alloc_pages_slowpath+0xcb2/0xcd0 [97259.192870] ll_ost_io04_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97259.193449] ll_ost_io01_018: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97259.193845] / [97259.195965] ? __alloc_pages_nodemask+0x166/0x330 [97259.197270] ,mems_allowed=0 [97259.198010] __alloc_pages_nodemask+0x2e2/0x330 [97259.201898] kmalloc_order+0x28/0x90 [97259.211953] kmalloc_order_trace+0x1d/0xb0 [97259.212925] __kmalloc+0x203/0x250 [97259.213778] ? __wake_up_common+0x7a/0x190 [97259.214744] virtqueue_add+0x493/0xc70 [97259.215668] ? finish_wait+0x80/0x80 [97259.216556] virtqueue_add_sgs+0x80/0xa0 [97259.217477] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97259.218631] ? scsi_alloc_sgtables+0x84/0x1c0 [97259.219613] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97259.220719] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97259.221919] scsi_queue_rq+0x512/0xb10 [97259.222806] __blk_mq_try_issue_directly+0x163/0x200 [97259.223890] blk_mq_request_issue_directly+0x4e/0xb0 [97259.225003] blk_mq_try_issue_list_directly+0x62/0x100 [97259.226148] blk_mq_sched_insert_requests+0xa4/0xf0 [97259.227211] blk_mq_flush_plug_list+0x135/0x220 [97259.228206] blk_flush_plug_list+0xd7/0x100 [97259.229146] blk_finish_plug+0x25/0x36 [97259.230035] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97259.231317] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97259.232469] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.233806] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.235110] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97259.236224] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97259.237284] ofd_commitrw+0x5b4/0xd20 [ofd] [97259.238239] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.239391] obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.240486] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97259.241631] ? flush_work+0x42/0x1d0 [97259.242466] ? internal_add_timer+0x42/0x70 [97259.243387] ? _cond_resched+0x15/0x30 [97259.244232] ? mutex_lock+0xe/0x30 [97259.245031] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97259.246189] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97259.247444] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97259.248714] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97259.249773] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97259.250924] kthread+0x134/0x150 [97259.251695] ? set_kthread_struct+0x50/0x50 [97259.252619] ret_from_fork+0x1f/0x40 [97259.253437] warn_alloc_show_mem: 1 callbacks suppressed [97259.253439] Mem-Info: [97259.253438] CPU: 17 PID: 527548 Comm: ll_ost_io04_036 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97259.254511] active_anon:27954 inactive_anon:199664 isolated_anon:0 active_file:32093469 inactive_file:3292402 isolated_file:0 unevictable:0 dirty:555 writeback:0 slab_reclaimable:1001036 slab_unreclaimable:604775 mapped:22732 shmem:146 pagetables:5697 bounce:0 free:793701 free_pcp:382 free_cma:0 [97259.255135] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97259.257486] Node 0 active_anon:111816kB inactive_anon:798656kB active_file:128408240kB inactive_file:13169608kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:90928kB dirty:2220kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35344kB pagetables:22788kB all_unreclaimable? no [97259.265198] Call Trace: [97259.266967] Node 0 [97259.272801] dump_stack+0x41/0x60 [97259.273469] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97259.274192] warn_alloc.cold.127+0x7b/0x108 [97259.274999] lowmem_reserve[]: 0 [97259.278650] __alloc_pages_slowpath+0xcb2/0xcd0 [97259.279614] 913 [97259.280468] ? __alloc_pages_nodemask+0x166/0x330 [97259.281494] 150007 [97259.282011] __alloc_pages_nodemask+0x2e2/0x330 [97259.283060] 150007 [97259.283640] kmalloc_order+0x28/0x90 [97259.284679] 150007 [97259.285239] kmalloc_order_trace+0x1d/0xb0 [97259.286700] __kmalloc+0x203/0x250 [97259.287680] Node 0 [97259.288180] virtqueue_add+0x493/0xc70 [97259.289043] DMA32 free:596608kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97259.289621] ? finish_wait+0x80/0x80 [97259.290558] lowmem_reserve[]: [97259.295079] virtqueue_add_sgs+0x80/0xa0 [97259.295980] 0 [97259.296814] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97259.297748] 0 [97259.298434] ? scsi_alloc_sgtables+0x84/0x1c0 [97259.299633] 149093 [97259.300295] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97259.301322] 149093 [97259.301955] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97259.303101] 149093 [97259.303872] scsi_queue_rq+0x512/0xb10 [97259.305721] __blk_mq_try_issue_directly+0x163/0x200 [97259.306651] Node 0 [97259.307216] blk_mq_request_issue_directly+0x4e/0xb0 [97259.308350] Normal free:2515604kB min:49268kB low:201940kB high:354612kB active_anon:111436kB inactive_anon:798652kB active_file:128430052kB inactive_file:13163444kB unevictable:0kB writepending:2220kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1936kB local_pcp:112kB free_cma:0kB [97259.308961] blk_mq_try_issue_list_directly+0x62/0x100 [97259.310096] lowmem_reserve[]: 0 [97259.315233] blk_mq_sched_insert_requests+0xa4/0xf0 [97259.316403] 0 [97259.317344] blk_mq_flush_plug_list+0x135/0x220 [97259.318432] 0 [97259.319142] blk_flush_plug_list+0xd7/0x100 [97259.320196] 0 [97259.320750] blk_finish_plug+0x25/0x36 [97259.321717] 0 [97259.322310] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97259.323726] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97259.324964] Node 0 [97259.325591] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.326706] DMA: [97259.327335] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.328631] 0*4kB [97259.329278] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97259.330546] 0*8kB [97259.331269] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97259.332384] 0*16kB [97259.332957] ofd_commitrw+0x5b4/0xd20 [ofd] [97259.334008] 0*32kB [97259.334570] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.335505] 0*64kB [97259.336062] obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.337062] 0*128kB [97259.337664] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97259.338636] 0*256kB [97259.339235] ? flush_work+0x42/0x1d0 [97259.340290] 0*512kB [97259.340874] ? internal_add_timer+0x42/0x70 [97259.341736] 1*1024kB [97259.342321] ? _cond_resched+0x15/0x30 [97259.343276] (U) [97259.343874] ? mutex_lock+0xe/0x30 [97259.344770] 1*2048kB [97259.345303] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97259.346114] (M) [97259.346707] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97259.347768] 2*4096kB [97259.348306] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97259.349490] (M) [97259.350163] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97259.351308] = 11264kB [97259.351833] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97259.352796] Node 0 [97259.353461] kthread+0x134/0x150 [97259.354545] DMA32: [97259.355117] ? set_kthread_struct+0x50/0x50 [97259.355903] 18*4kB [97259.356421] ret_from_fork+0x1f/0x40 [97259.357350] (UM) [97259.357878] CPU: 7 PID: 527075 Comm: ll_ost_io01_018 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97259.358653] 21*8kB (UMH) [97259.359376] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97259.361762] 25*16kB (UME) [97259.362607] Call Trace: [97259.364341] 51*32kB (UMH) [97259.365223] dump_stack+0x41/0x60 [97259.365886] 46*64kB [97259.366738] warn_alloc.cold.127+0x7b/0x108 [97259.367507] (UMH) [97259.368252] __alloc_pages_slowpath+0xcb2/0xcd0 [97259.369152] 46*128kB [97259.369871] ? __alloc_pages_nodemask+0x166/0x330 [97259.370838] (UM) [97259.371590] __alloc_pages_nodemask+0x2e2/0x330 [97259.372592] 44*256kB [97259.373313] kmalloc_order+0x28/0x90 [97259.374250] (UM) [97259.375023] kmalloc_order_trace+0x1d/0xb0 [97259.375818] 46*512kB [97259.376566] __kmalloc+0x203/0x250 [97259.377460] (UMEH) [97259.378227] ? __wake_up_common+0x7a/0x190 [97259.379036] 38*1024kB [97259.379798] virtqueue_add+0x493/0xc70 [97259.380698] (UM) [97259.381465] ? finish_wait+0x80/0x80 [97259.382296] 42*2048kB [97259.382986] virtqueue_add_sgs+0x80/0xa0 [97259.383821] (UM) [97259.384621] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97259.385500] 104*4096kB [97259.386217] ? scsi_alloc_sgtables+0x84/0x1c0 [97259.387344] (UM) [97259.388170] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97259.389142] = 596832kB [97259.389870] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97259.390949] Node 0 [97259.391739] scsi_queue_rq+0x512/0xb10 [97259.392925] Normal: [97259.393655] __blk_mq_try_issue_directly+0x163/0x200 [97259.394521] 285273*4kB [97259.395253] blk_mq_request_issue_directly+0x4e/0xb0 [97259.396303] (UME) [97259.397111] blk_mq_try_issue_list_directly+0x62/0x100 [97259.398161] 158888*8kB [97259.398885] blk_mq_sched_insert_requests+0xa4/0xf0 [97259.399971] (UME) [97259.400768] blk_mq_flush_plug_list+0x135/0x220 [97259.401808] 4718*16kB [97259.402547] blk_flush_plug_list+0xd7/0x100 [97259.403469] (UMEH) 7*32kB [97259.404264] blk_finish_plug+0x25/0x36 [97259.405178] (H) [97259.406032] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97259.406833] 0*64kB [97259.407518] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97259.408673] 0*128kB [97259.409396] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.410463] 0*256kB [97259.411224] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.412464] 0*512kB [97259.413201] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97259.414430] 0*1024kB [97259.415175] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97259.416237] 0*2048kB [97259.417106] ofd_commitrw+0x5b4/0xd20 [ofd] [97259.418098] 0*4096kB [97259.418862] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.419723] = 2487908kB [97259.420484] obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.421431] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97259.422219] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97259.423119] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97259.425100] ? flush_work+0x42/0x1d0 [97259.426104] 35410061 total pagecache pages [97259.428045] ? internal_add_timer+0x42/0x70 [97259.428876] 1570 pages in swap cache [97259.430007] ? _cond_resched+0x15/0x30 [97259.430934] Swap cache stats: add 1606821, delete 1605265, find 83330/100457 [97259.431981] ? mutex_lock+0xe/0x30 [97259.432872] Free swap = 5334092kB [97259.434578] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97259.435369] Total swap = 11075580kB [97259.436382] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97259.437451] 39321462 pages RAM [97259.438463] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97259.439628] 0 pages HighMem/MovableOnly [97259.440575] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97259.441733] 905848 pages reserved [97259.442816] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97259.443776] 0 pages hwpoisoned [97259.444755] kthread+0x134/0x150 [97259.447618] ? set_kthread_struct+0x50/0x50 [97259.448719] ret_from_fork+0x1f/0x40 [97259.449695] CPU: 5 PID: 527166 Comm: ll_ost_io01_060 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97259.452752] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97259.455443] Call Trace: [97259.456283] dump_stack+0x41/0x60 [97259.457206] warn_alloc.cold.127+0x7b/0x108 [97259.458357] __alloc_pages_slowpath+0xcb2/0xcd0 [97259.459505] ? __alloc_pages_nodemask+0x166/0x330 [97259.460691] __alloc_pages_nodemask+0x2e2/0x330 [97259.461868] kmalloc_order+0x28/0x90 [97259.462858] kmalloc_order_trace+0x1d/0xb0 [97259.463923] __kmalloc+0x203/0x250 [97259.464886] virtqueue_add+0x493/0xc70 [97259.465889] virtqueue_add_sgs+0x80/0xa0 [97259.466959] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97259.468306] ? scsi_alloc_sgtables+0x84/0x1c0 [97259.469439] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97259.470684] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97259.472057] scsi_queue_rq+0x512/0xb10 [97259.473062] __blk_mq_try_issue_directly+0x163/0x200 [97259.474291] blk_mq_request_issue_directly+0x4e/0xb0 [97259.475537] blk_mq_try_issue_list_directly+0x62/0x100 [97259.476807] blk_mq_sched_insert_requests+0xa4/0xf0 [97259.478035] blk_mq_flush_plug_list+0x135/0x220 [97259.479209] blk_flush_plug_list+0xd7/0x100 [97259.480296] blk_finish_plug+0x25/0x36 [97259.481292] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97259.482723] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97259.484049] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.485583] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.487079] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97259.488403] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97259.489626] ofd_commitrw+0x5b4/0xd20 [ofd] [97259.490712] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.491999] obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.493249] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97259.494562] ? flush_work+0x42/0x1d0 [97259.495523] ? internal_add_timer+0x42/0x70 [97259.496601] ? _cond_resched+0x15/0x30 [97259.497623] ? mutex_lock+0xe/0x30 [97259.498571] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97259.499962] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97259.501447] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97259.502957] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97259.504227] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97259.505587] kthread+0x134/0x150 [97259.506501] ? set_kthread_struct+0x50/0x50 [97259.507583] ret_from_fork+0x1f/0x40 [97259.508553] CPU: 18 PID: 527998 Comm: ll_ost_io04_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97259.511161] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97259.513294] Call Trace: [97259.514093] dump_stack+0x41/0x60 [97259.515059] warn_alloc.cold.127+0x7b/0x108 [97259.516188] __alloc_pages_slowpath+0xcb2/0xcd0 [97259.517373] ? __alloc_pages_nodemask+0x166/0x330 [97259.518586] __alloc_pages_nodemask+0x2e2/0x330 [97259.519783] kmalloc_order+0x28/0x90 [97259.520777] kmalloc_order_trace+0x1d/0xb0 [97259.521869] __kmalloc+0x203/0x250 [97259.522824] virtqueue_add+0x493/0xc70 [97259.523837] virtqueue_add_sgs+0x80/0xa0 [97259.524896] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97259.526244] ? scsi_alloc_sgtables+0x84/0x1c0 [97259.527383] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97259.528666] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97259.530048] scsi_queue_rq+0x512/0xb10 [97259.531048] __blk_mq_try_issue_directly+0x163/0x200 [97259.532316] blk_mq_request_issue_directly+0x4e/0xb0 [97259.533563] blk_mq_try_issue_list_directly+0x62/0x100 [97259.534827] blk_mq_sched_insert_requests+0xa4/0xf0 [97259.536052] blk_mq_flush_plug_list+0x135/0x220 [97259.537219] blk_flush_plug_list+0xd7/0x100 [97259.538307] blk_finish_plug+0x25/0x36 [97259.539313] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97259.540768] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97259.542092] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.543626] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.545119] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97259.546410] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97259.547638] ofd_commitrw+0x5b4/0xd20 [ofd] [97259.548734] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.550042] obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.551295] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97259.552574] ? flush_work+0x42/0x1d0 [97259.553438] ? internal_add_timer+0x42/0x70 [97259.554286] ? _cond_resched+0x15/0x30 [97259.555110] ? mutex_lock+0xe/0x30 [97259.555981] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97259.557379] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97259.558879] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97259.560412] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97259.561655] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97259.563015] kthread+0x134/0x150 [97259.563907] ? set_kthread_struct+0x50/0x50 [97259.564967] ret_from_fork+0x1f/0x40 [97259.565907] CPU: 11 PID: 527865 Comm: ll_ost_io02_074 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97259.568664] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97259.570789] Call Trace: [97259.571435] dump_stack+0x41/0x60 [97259.572259] warn_alloc.cold.127+0x7b/0x108 [97259.573408] __alloc_pages_slowpath+0xcb2/0xcd0 [97259.574611] ? __alloc_pages_nodemask+0x166/0x330 [97259.575850] __alloc_pages_nodemask+0x2e2/0x330 [97259.577030] kmalloc_order+0x28/0x90 [97259.578170] kmalloc_order_trace+0x1d/0xb0 [97259.579305] __kmalloc+0x203/0x250 [97259.580372] ? __wake_up_common+0x7a/0x190 [97259.581562] virtqueue_add+0x493/0xc70 [97259.582704] ? finish_wait+0x80/0x80 [97259.583807] virtqueue_add_sgs+0x80/0xa0 [97259.584949] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97259.586396] ? scsi_alloc_sgtables+0x84/0x1c0 [97259.587611] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97259.588950] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97259.590431] scsi_queue_rq+0x512/0xb10 [97259.591592] __blk_mq_try_issue_directly+0x163/0x200 [97259.592948] blk_mq_request_issue_directly+0x4e/0xb0 [97259.594324] blk_mq_try_issue_list_directly+0x62/0x100 [97259.595766] blk_mq_sched_insert_requests+0xa4/0xf0 [97259.597109] blk_mq_flush_plug_list+0x135/0x220 [97259.598361] blk_flush_plug_list+0xd7/0x100 [97259.599589] blk_finish_plug+0x25/0x36 [97259.600674] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97259.602257] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97259.603717] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.605353] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97259.606969] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97259.608389] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97259.609695] ofd_commitrw+0x5b4/0xd20 [ofd] [97259.610885] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.612269] obd_commitrw+0x1b6/0x370 [ptlrpc] [97259.613626] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97259.615017] ? flush_work+0x42/0x1d0 [97259.616097] ? internal_add_timer+0x42/0x70 [97259.617225] ? _cond_resched+0x15/0x30 [97259.618327] ? mutex_lock+0xe/0x30 [97259.619306] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97259.620731] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97259.622272] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97259.623808] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97259.625084] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97259.626476] kthread+0x134/0x150 [97259.627396] ? set_kthread_struct+0x50/0x50 [97259.628557] ret_from_fork+0x1f/0x40 [97263.475216] ll_ost_io01_051: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97263.475218] ll_ost_io02_018: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97263.475225] ll_ost_io02_046: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97263.475237] CPU: 11 PID: 527251 Comm: ll_ost_io02_046 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97263.475241] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97263.475243] Call Trace: [97263.475247] dump_stack+0x41/0x60 [97263.475256] warn_alloc.cold.127+0x7b/0x108 [97263.475264] __alloc_pages_slowpath+0xcb2/0xcd0 [97263.475268] ? __alloc_pages_nodemask+0x166/0x330 [97263.475271] __alloc_pages_nodemask+0x2e2/0x330 [97263.475275] kmalloc_order+0x28/0x90 [97263.475281] kmalloc_order_trace+0x1d/0xb0 [97263.475281] ll_ost_io01_081: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97263.475285] __kmalloc+0x203/0x250 [97263.475291] virtqueue_add+0x493/0xc70 [97263.475297] ? finish_wait+0x80/0x80 [97263.475302] virtqueue_add_sgs+0x80/0xa0 [97263.475308] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97263.475314] ? scsi_alloc_sgtables+0x84/0x1c0 [97263.475319] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97263.475322] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97263.475326] scsi_queue_rq+0x512/0xb10 [97263.475328] __blk_mq_try_issue_directly+0x163/0x200 [97263.475335] blk_mq_request_issue_directly+0x4e/0xb0 [97263.475338] blk_mq_try_issue_list_directly+0x62/0x100 [97263.475342] blk_mq_sched_insert_requests+0xa4/0xf0 [97263.475346] blk_mq_flush_plug_list+0x135/0x220 [97263.475350] blk_flush_plug_list+0xd7/0x100 [97263.475356] blk_finish_plug+0x25/0x36 [97263.475361] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97263.475398] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97263.475434] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.475460] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.475490] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97263.475519] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97263.475540] ofd_commitrw+0x5b4/0xd20 [ofd] [97263.475559] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.475695] obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.475812] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97263.475928] ? flush_work+0x42/0x1d0 [97263.475936] ? internal_add_timer+0x42/0x70 [97263.475940] ? _cond_resched+0x15/0x30 [97263.475944] ? mutex_lock+0xe/0x30 [97263.475948] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97263.476060] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97263.476181] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97263.476299] ? lprocfs_counter_add+0x86/0x140 [obdclass] [97263.476374] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97263.476497] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97263.476607] kthread+0x134/0x150 [97263.476611] ? set_kthread_struct+0x50/0x50 [97263.476615] ret_from_fork+0x1f/0x40 [97263.476622] warn_alloc_show_mem: 5 callbacks suppressed [97263.476624] Mem-Info: [97263.476623] CPU: 5 PID: 528150 Comm: ll_ost_io01_081 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97263.476627] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97263.476629] Call Trace: [97263.476627] active_anon:27954 inactive_anon:199545 isolated_anon:0 active_file:31594683 inactive_file:3142024 isolated_file:0 unevictable:0 dirty:1642 writeback:26 slab_reclaimable:1001031 slab_unreclaimable:604658 mapped:22893 shmem:146 pagetables:5696 bounce:0 free:1442819 free_pcp:461 free_cma:0 [97263.476634] Node 0 active_anon:111816kB inactive_anon:798180kB active_file:126378732kB inactive_file:12568096kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:91572kB dirty:6568kB writeback:104kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35360kB pagetables:22784kB all_unreclaimable? no [97263.476633] dump_stack+0x41/0x60 [97263.476642] Node 0 [97263.476641] warn_alloc.cold.127+0x7b/0x108 [97263.476643] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97263.476647] __alloc_pages_slowpath+0xcb2/0xcd0 [97263.476649] lowmem_reserve[]: 0 913 150007 150007 [97263.476652] ? __alloc_pages_nodemask+0x166/0x330 [97263.476653] 150007 [97263.476656] Node 0 [97263.476656] __alloc_pages_nodemask+0x2e2/0x330 [97263.476657] DMA32 free:596480kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97263.476659] kmalloc_order+0x28/0x90 [97263.476663] lowmem_reserve[]: 0 0 149093 149093 [97263.476665] kmalloc_order_trace+0x1d/0xb0 [97263.476667] 149093 [97263.476669] Node 0 [97263.476669] __kmalloc+0x203/0x250 [97263.476670] Normal free:5163560kB min:49268kB low:201940kB high:354612kB active_anon:111436kB inactive_anon:798176kB active_file:126379052kB inactive_file:12568328kB unevictable:0kB writepending:6672kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1896kB local_pcp:52kB free_cma:0kB [97263.476677] lowmem_reserve[]: 0 0 0 [97263.476675] virtqueue_add+0x493/0xc70 [97263.476680] 0 0 [97263.476699] Node 0 DMA: 0*4kB [97263.476699] ? finish_wait+0x80/0x80 [97263.476702] 0*8kB 0*16kB 0*32kB [97263.476704] virtqueue_add_sgs+0x80/0xa0 [97263.476705] 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB [97263.476710] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97263.476713] (M) = 11264kB [97263.476715] Node 0 DMA32: 18*4kB (UM) 21*8kB (UMH) [97263.476716] ? scsi_alloc_sgtables+0x84/0x1c0 [97263.476719] 25*16kB (UME) 43*32kB (UMH) 44*64kB [97263.476722] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97263.476724] (UM) 46*128kB (UM) 44*256kB (UM) [97263.476726] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97263.476728] 46*512kB (UMEH) 38*1024kB [97263.476730] scsi_queue_rq+0x512/0xb10 [97263.476731] (UM) 42*2048kB (UM) 104*4096kB (UM) = 596448kB [97263.476737] Node 0 [97263.476734] __blk_mq_try_issue_directly+0x163/0x200 [97263.476738] Normal: 707296*4kB (UME) 276031*8kB (UME) [97263.476741] blk_mq_request_issue_directly+0x4e/0xb0 [97263.476743] 7818*16kB (UMEH) 0*32kB 0*64kB [97263.476745] blk_mq_try_issue_list_directly+0x62/0x100 [97263.476747] 0*128kB 0*256kB 0*512kB 0*1024kB [97263.476750] blk_mq_sched_insert_requests+0xa4/0xf0 [97263.476752] 0*2048kB 0*4096kB = 5162520kB [97263.476754] blk_mq_flush_plug_list+0x135/0x220 [97263.476756] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97263.476759] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97263.476761] 34736885 total pagecache pages [97263.476759] blk_flush_plug_list+0xd7/0x100 [97263.476761] 1591 pages in swap cache [97263.476763] Swap cache stats: add 1606977, delete 1605409, find 83331/100458 [97263.476765] Free swap = 5333316kB [97263.476766] Total swap = 11075580kB [97263.476765] blk_finish_plug+0x25/0x36 [97263.476767] 39321462 pages RAM [97263.476768] 0 pages HighMem/MovableOnly [97263.476769] 905848 pages reserved [97263.476770] 0 pages hwpoisoned [97263.476771] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97263.476810] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97263.476846] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.476876] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.476905] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97263.476935] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97263.476956] ofd_commitrw+0x5b4/0xd20 [ofd] [97263.476975] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.477101] obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.477216] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97263.477333] ? flush_work+0x42/0x1d0 [97263.477338] ? internal_add_timer+0x42/0x70 [97263.477342] ? _cond_resched+0x15/0x30 [97263.477345] ? mutex_lock+0xe/0x30 [97263.477349] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97263.477501] ,cpuset= [97263.477465] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97263.477572] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97263.477692] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97263.477802] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97263.477911] kthread+0x134/0x150 [97263.477915] ? set_kthread_struct+0x50/0x50 [97263.477918] ret_from_fork+0x1f/0x40 [97263.480228] /,mems_allowed=0 [97263.652949] /,mems_allowed=0 [97263.653514] CPU: 9 PID: 527170 Comm: ll_ost_io02_018 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97263.656726] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97263.658227] Call Trace: [97263.658767] dump_stack+0x41/0x60 [97263.659421] warn_alloc.cold.127+0x7b/0x108 [97263.660196] __alloc_pages_slowpath+0xcb2/0xcd0 [97263.661020] ? __alloc_pages_nodemask+0x166/0x330 [97263.661885] __alloc_pages_nodemask+0x2e2/0x330 [97263.662713] kmalloc_order+0x28/0x90 [97263.663405] kmalloc_order_trace+0x1d/0xb0 [97263.664149] __kmalloc+0x203/0x250 [97263.664803] virtqueue_add+0x493/0xc70 [97263.665513] ? finish_wait+0x80/0x80 [97263.666196] virtqueue_add_sgs+0x80/0xa0 [97263.666934] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97263.667870] ? scsi_alloc_sgtables+0x84/0x1c0 [97263.668659] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97263.669538] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97263.670530] scsi_queue_rq+0x512/0xb10 [97263.671222] __blk_mq_try_issue_directly+0x163/0x200 [97263.672120] blk_mq_request_issue_directly+0x4e/0xb0 [97263.673020] blk_mq_try_issue_list_directly+0x62/0x100 [97263.673942] blk_mq_sched_insert_requests+0xa4/0xf0 [97263.674819] blk_mq_flush_plug_list+0x135/0x220 [97263.675646] blk_flush_plug_list+0xd7/0x100 [97263.676485] blk_finish_plug+0x25/0x36 [97263.677191] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97263.678216] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97263.679152] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.680244] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.681301] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97263.682340] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97263.683434] ofd_commitrw+0x5b4/0xd20 [ofd] [97263.684219] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.685199] obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.686118] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97263.687034] ? flush_work+0x42/0x1d0 [97263.687737] ? internal_add_timer+0x42/0x70 [97263.688522] ? _cond_resched+0x15/0x30 [97263.689247] ? mutex_lock+0xe/0x30 [97263.689920] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97263.690911] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97263.691973] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97263.693078] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97263.694021] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97263.695109] kthread+0x134/0x150 [97263.695847] ? set_kthread_struct+0x50/0x50 [97263.696749] ret_from_fork+0x1f/0x40 [97263.697682] CPU: 4 PID: 527155 Comm: ll_ost_io01_051 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97263.700404] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97263.702211] Call Trace: [97263.702804] dump_stack+0x41/0x60 [97263.703640] warn_alloc.cold.127+0x7b/0x108 [97263.704662] __alloc_pages_slowpath+0xcb2/0xcd0 [97263.705802] ? __alloc_pages_nodemask+0x166/0x330 [97263.706875] __alloc_pages_nodemask+0x2e2/0x330 [97263.707896] kmalloc_order+0x28/0x90 [97263.708655] kmalloc_order_trace+0x1d/0xb0 [97263.709485] __kmalloc+0x203/0x250 [97263.710189] ? mempool_alloc+0x67/0x180 [97263.710970] virtqueue_add+0x493/0xc70 [97263.711915] ? finish_wait+0x80/0x80 [97263.712835] virtqueue_add_sgs+0x80/0xa0 [97263.713633] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97263.714613] ? scsi_alloc_sgtables+0x84/0x1c0 [97263.715458] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97263.716380] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97263.717362] scsi_queue_rq+0x512/0xb10 [97263.718099] __blk_mq_try_issue_directly+0x163/0x200 [97263.719026] blk_mq_request_issue_directly+0x4e/0xb0 [97263.719945] blk_mq_try_issue_list_directly+0x62/0x100 [97263.721154] blk_mq_sched_insert_requests+0xa4/0xf0 [97263.722099] blk_mq_flush_plug_list+0x135/0x220 [97263.722957] blk_flush_plug_list+0xd7/0x100 [97263.723738] blk_finish_plug+0x25/0x36 [97263.724455] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97263.725540] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97263.726560] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.727699] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97263.728994] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97263.730247] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97263.731471] ofd_commitrw+0x5b4/0xd20 [ofd] [97263.732475] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.733674] obd_commitrw+0x1b6/0x370 [ptlrpc] [97263.734794] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97263.735965] ? flush_work+0x42/0x1d0 [97263.736881] ? internal_add_timer+0x42/0x70 [97263.737719] ? _cond_resched+0x15/0x30 [97263.738527] ? mutex_lock+0xe/0x30 [97263.739260] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97263.740466] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97263.741845] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97263.742998] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97263.743966] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97263.744980] kthread+0x134/0x150 [97263.745646] ? set_kthread_struct+0x50/0x50 [97263.746438] ret_from_fork+0x1f/0x40 [97265.043415] warn_alloc: 35 callbacks suppressed [97265.043419] ll_ost_io01_056: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97265.043446] ll_ost_io04_058: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97265.043455] ll_ost_io04_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97265.043466] CPU: 18 PID: 527193 Comm: ll_ost_io04_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.043469] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.043471] Call Trace: [97265.043470] ll_ost_io01_000: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97265.043476] dump_stack+0x41/0x60 [97265.043485] warn_alloc.cold.127+0x7b/0x108 [97265.043492] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.043495] ? __alloc_pages_nodemask+0x166/0x330 [97265.043499] __alloc_pages_nodemask+0x2e2/0x330 [97265.043502] kmalloc_order+0x28/0x90 [97265.043508] kmalloc_order_trace+0x1d/0xb0 [97265.043512] __kmalloc+0x203/0x250 [97265.043518] virtqueue_add+0x493/0xc70 [97265.043524] virtqueue_add_sgs+0x80/0xa0 [97265.043529] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.043535] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.043541] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.043544] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.043547] scsi_queue_rq+0x512/0xb10 [97265.043550] __blk_mq_try_issue_directly+0x163/0x200 [97265.043557] blk_mq_request_issue_directly+0x4e/0xb0 [97265.043561] blk_mq_try_issue_list_directly+0x62/0x100 [97265.043564] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.043569] blk_mq_flush_plug_list+0x135/0x220 [97265.043572] blk_flush_plug_list+0xd7/0x100 [97265.043578] blk_finish_plug+0x25/0x36 [97265.043583] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.043623] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.043656] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.043682] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.043709] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.043736] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.043755] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.043773] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.043897] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.044006] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.044109] ? newidle_balance+0x2b6/0x3b0 [97265.044114] ? internal_add_timer+0x42/0x70 [97265.044117] ? _cond_resched+0x15/0x30 [97265.044120] ? mutex_lock+0xe/0x30 [97265.044123] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.044227] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.044333] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.044434] ll_ost_io04_029: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97265.044439] ? finish_wait+0x80/0x80 [97265.044440] ,cpuset=/,mems_allowed=0 [97265.044442] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.044548] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.044659] kthread+0x134/0x150 [97265.044664] ? set_kthread_struct+0x50/0x50 [97265.044667] ret_from_fork+0x1f/0x40 [97265.044672] warn_alloc_show_mem: 3 callbacks suppressed [97265.044673] Mem-Info: [97265.044672] CPU: 17 PID: 527255 Comm: ll_ost_io04_029 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.044677] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.044676] active_anon:27954 inactive_anon:199533 isolated_anon:0 active_file:31767918 inactive_file:3016964 isolated_file:0 unevictable:0 dirty:179 writeback:3 slab_reclaimable:1001027 slab_unreclaimable:604786 mapped:22946 shmem:146 pagetables:5696 bounce:0 free:1394434 free_pcp:403 free_cma:0 [97265.044678] Call Trace: [97265.044682] Node 0 active_anon:111816kB inactive_anon:798132kB active_file:127071672kB inactive_file:12068340kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:91784kB dirty:716kB writeback:12kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35360kB pagetables:22784kB all_unreclaimable? no [97265.044683] dump_stack+0x41/0x60 [97265.044688] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97265.044691] warn_alloc.cold.127+0x7b/0x108 [97265.044694] lowmem_reserve[]: 0 913 150007 150007 150007 [97265.044699] Node 0 [97265.044698] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.044700] DMA32 free:596512kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97265.044703] ? __alloc_pages_nodemask+0x166/0x330 [97265.044706] lowmem_reserve[]: 0 0 [97265.044706] __alloc_pages_nodemask+0x2e2/0x330 [97265.044707] 149093 149093 149093 [97265.044710] Node 0 Normal free:4969988kB min:49268kB low:201940kB high:354612kB active_anon:111436kB inactive_anon:798128kB active_file:127071320kB inactive_file:12068336kB unevictable:0kB writepending:728kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1564kB local_pcp:32kB free_cma:0kB [97265.044710] kmalloc_order+0x28/0x90 [97265.044716] lowmem_reserve[]: 0 0 [97265.044716] kmalloc_order_trace+0x1d/0xb0 [97265.044718] 0 0 0 [97265.044721] Node 0 DMA: 0*4kB 0*8kB [97265.044720] __kmalloc+0x203/0x250 [97265.044724] 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB [97265.044726] virtqueue_add+0x493/0xc70 [97265.044730] (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [97265.044734] Node 0 [97265.044733] virtqueue_add_sgs+0x80/0xa0 [97265.044735] DMA32: 18*4kB (UM) 21*8kB (UMH) 25*16kB (UME) 42*32kB (UM) 45*64kB (UMH) [97265.044739] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.044743] 46*128kB (UM) 44*256kB (UM) 46*512kB (UMEH) 38*1024kB (UM) 42*2048kB (UM) 104*4096kB [97265.044746] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.044750] (UM) = 596480kB [97265.044752] Node 0 Normal: 558108*4kB (UME) 323910*8kB [97265.044754] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.044756] (UME) 9094*16kB (UMEH) 0*32kB 0*64kB 0*128kB [97265.044758] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.044761] 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB [97265.044763] scsi_queue_rq+0x512/0xb10 [97265.044765] = 4969216kB [97265.044768] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97265.044770] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97265.044771] 34785018 total pagecache pages [97265.044767] __blk_mq_try_issue_directly+0x163/0x200 [97265.044772] 1578 pages in swap cache [97265.044773] Swap cache stats: add 1606977, delete 1605422, find 83331/100458 [97265.044775] Free swap = 5333316kB [97265.044775] Total swap = 11075580kB [97265.044776] 39321462 pages RAM [97265.044775] blk_mq_request_issue_directly+0x4e/0xb0 [97265.044777] 0 pages HighMem/MovableOnly [97265.044778] 905848 pages reserved [97265.044779] 0 pages hwpoisoned [97265.044780] blk_mq_try_issue_list_directly+0x62/0x100 [97265.044785] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.044790] blk_mq_flush_plug_list+0x135/0x220 [97265.044794] blk_flush_plug_list+0xd7/0x100 [97265.044799] blk_finish_plug+0x25/0x36 [97265.044805] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.044845] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.044871] ,cpuset= [97265.044879] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.044920] ll_ost_io02_083: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97265.044909] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.044926] ,cpuset=/,mems_allowed=0 [97265.044942] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.044971] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.044999] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.045014] ll_ost_io01_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97265.045025] ll_ost_io04_048: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97265.045020] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.045158] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.045274] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.045397] ? flush_work+0x42/0x1d0 [97265.045404] ? internal_add_timer+0x42/0x70 [97265.045408] ? _cond_resched+0x15/0x30 [97265.045411] ? mutex_lock+0xe/0x30 [97265.045415] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.045538] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.045659] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.045776] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.045901] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.046016] kthread+0x134/0x150 [97265.046021] ? set_kthread_struct+0x50/0x50 [97265.046024] ret_from_fork+0x1f/0x40 [97265.046031] CPU: 5 PID: 527199 Comm: ll_ost_io01_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.046035] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.046037] Call Trace: [97265.046040] dump_stack+0x41/0x60 [97265.046047] warn_alloc.cold.127+0x7b/0x108 [97265.046052] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.046056] ? __alloc_pages_nodemask+0x166/0x330 [97265.046059] __alloc_pages_nodemask+0x2e2/0x330 [97265.046062] kmalloc_order+0x28/0x90 [97265.046067] kmalloc_order_trace+0x1d/0xb0 [97265.046071] __kmalloc+0x203/0x250 [97265.046076] virtqueue_add+0x493/0xc70 [97265.046082] ? finish_wait+0x80/0x80 [97265.046086] virtqueue_add_sgs+0x80/0xa0 [97265.046090] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.046096] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.046101] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.046104] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.046107] scsi_queue_rq+0x512/0xb10 [97265.046110] __blk_mq_try_issue_directly+0x163/0x200 [97265.046117] blk_mq_request_issue_directly+0x4e/0xb0 [97265.046121] blk_mq_try_issue_list_directly+0x62/0x100 [97265.046124] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.046129] blk_mq_flush_plug_list+0x135/0x220 [97265.046133] blk_flush_plug_list+0xd7/0x100 [97265.046138] blk_finish_plug+0x25/0x36 [97265.046142] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.046177] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.046207] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.046233] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.046259] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.046285] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.046304] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.046322] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.046442] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.046551] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.046659] ? flush_work+0x42/0x1d0 [97265.046663] ? _cond_resched+0x15/0x30 [97265.046666] ? mutex_lock+0xe/0x30 [97265.046668] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.046778] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.046887] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.047000] ? finish_wait+0x80/0x80 [97265.047003] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.047124] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.047236] kthread+0x134/0x150 [97265.047240] ? set_kthread_struct+0x50/0x50 [97265.047243] ret_from_fork+0x1f/0x40 [97265.047249] CPU: 4 PID: 188100 Comm: ll_ost_io01_000 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.047252] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.047253] Call Trace: [97265.047255] dump_stack+0x41/0x60 [97265.047258] ,cpuset= [97265.047261] warn_alloc.cold.127+0x7b/0x108 [97265.047267] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.047271] __alloc_pages_nodemask+0x2e2/0x330 [97265.047274] kmalloc_order+0x28/0x90 [97265.047280] kmalloc_order_trace+0x1d/0xb0 [97265.047283] __kmalloc+0x203/0x250 [97265.047289] virtqueue_add+0x493/0xc70 [97265.047295] ? finish_wait+0x80/0x80 [97265.047298] virtqueue_add_sgs+0x80/0xa0 [97265.047302] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.047308] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.047314] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.047318] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.047322] scsi_queue_rq+0x512/0xb10 [97265.047325] __blk_mq_try_issue_directly+0x163/0x200 [97265.047332] blk_mq_request_issue_directly+0x4e/0xb0 [97265.047336] blk_mq_try_issue_list_directly+0x62/0x100 [97265.047340] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.047345] blk_mq_flush_plug_list+0x135/0x220 [97265.047350] blk_flush_plug_list+0xd7/0x100 [97265.047356] blk_finish_plug+0x25/0x36 [97265.047361] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.047399] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.047431] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.047461] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.047490] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.047519] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.047534] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.047545] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.047625] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.047694] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.047763] ? flush_work+0x42/0x1d0 [97265.047767] ? internal_add_timer+0x42/0x70 [97265.047770] ? _cond_resched+0x15/0x30 [97265.047771] ? mutex_lock+0xe/0x30 [97265.047773] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.047845] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.047916] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.047988] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.048060] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.048128] kthread+0x134/0x150 [97265.048132] ? set_kthread_struct+0x50/0x50 [97265.048135] ret_from_fork+0x1f/0x40 [97265.048141] CPU: 8 PID: 527876 Comm: ll_ost_io02_083 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.048146] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.048147] Call Trace: [97265.048150] dump_stack+0x41/0x60 [97265.048157] warn_alloc.cold.127+0x7b/0x108 [97265.048164] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.048168] ? __alloc_pages_nodemask+0x166/0x330 [97265.048172] __alloc_pages_nodemask+0x2e2/0x330 [97265.048176] kmalloc_order+0x28/0x90 [97265.048181] kmalloc_order_trace+0x1d/0xb0 [97265.048186] __kmalloc+0x203/0x250 [97265.048191] virtqueue_add+0x493/0xc70 [97265.048196] virtqueue_add_sgs+0x80/0xa0 [97265.048201] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.048207] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.048213] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.048217] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.048221] scsi_queue_rq+0x512/0xb10 [97265.048225] __blk_mq_try_issue_directly+0x163/0x200 [97265.048231] blk_mq_request_issue_directly+0x4e/0xb0 [97265.048236] blk_mq_try_issue_list_directly+0x62/0x100 [97265.048240] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.048244] blk_mq_flush_plug_list+0x135/0x220 [97265.048249] blk_flush_plug_list+0xd7/0x100 [97265.048254] blk_finish_plug+0x25/0x36 [97265.048259] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.048293] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.048324] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.048352] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.048382] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.048411] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.048432] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.048451] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.048576] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.048706] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.048836] ? flush_work+0x42/0x1d0 [97265.048842] ? internal_add_timer+0x42/0x70 [97265.048846] ? _cond_resched+0x15/0x30 [97265.048848] ? mutex_lock+0xe/0x30 [97265.048852] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.048969] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.049084] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.049195] ? tgt_key_exit+0xc/0x20 [ptlrpc] [97265.049322] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.049446] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.049558] kthread+0x134/0x150 [97265.049562] ? set_kthread_struct+0x50/0x50 [97265.049565] ret_from_fork+0x1f/0x40 [97265.049572] CPU: 18 PID: 527685 Comm: ll_ost_io04_048 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.049576] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.049577] Call Trace: [97265.049580] dump_stack+0x41/0x60 [97265.049585] warn_alloc.cold.127+0x7b/0x108 [97265.049590] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.049594] ? __alloc_pages_nodemask+0x166/0x330 [97265.049597] __alloc_pages_nodemask+0x2e2/0x330 [97265.049599] kmalloc_order+0x28/0x90 [97265.049604] kmalloc_order_trace+0x1d/0xb0 [97265.049607] __kmalloc+0x203/0x250 [97265.049611] ? __wake_up_common+0x7a/0x190 [97265.049615] virtqueue_add+0x493/0xc70 [97265.049620] ? finish_wait+0x80/0x80 [97265.049623] virtqueue_add_sgs+0x80/0xa0 [97265.049627] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.049632] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.049637] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.049640] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.049643] scsi_queue_rq+0x512/0xb10 [97265.049646] __blk_mq_try_issue_directly+0x163/0x200 [97265.049652] blk_mq_request_issue_directly+0x4e/0xb0 [97265.049655] blk_mq_try_issue_list_directly+0x62/0x100 [97265.049659] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.049662] blk_mq_flush_plug_list+0x135/0x220 [97265.049667] blk_flush_plug_list+0xd7/0x100 [97265.049671] blk_finish_plug+0x25/0x36 [97265.049676] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.049708] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [97265.049735] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.049760] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.049786] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.049812] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.049829] / [97265.049831] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.049849] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.049972] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.050102] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.050231] ? newidle_balance+0x2b6/0x3b0 [97265.050236] ? internal_add_timer+0x42/0x70 [97265.050239] ? _cond_resched+0x15/0x30 [97265.050242] ? mutex_lock+0xe/0x30 [97265.050245] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.050366] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.050487] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.050613] ? finish_wait+0x80/0x80 [97265.050617] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.050743] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.050870] kthread+0x134/0x150 [97265.050874] ? set_kthread_struct+0x50/0x50 [97265.050878] ret_from_fork+0x1f/0x40 [97265.055388] ll_ost_io00_056: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97265.055867] ,mems_allowed=0 [97265.055868] / [97265.055873] CPU: 7 PID: 527160 Comm: ll_ost_io01_056 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.055877] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.055879] Call Trace: [97265.055882] dump_stack+0x41/0x60 [97265.055889] warn_alloc.cold.127+0x7b/0x108 [97265.055894] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.055898] ? __alloc_pages_nodemask+0x166/0x330 [97265.055901] __alloc_pages_nodemask+0x2e2/0x330 [97265.055905] kmalloc_order+0x28/0x90 [97265.055910] kmalloc_order_trace+0x1d/0xb0 [97265.055913] __kmalloc+0x203/0x250 [97265.055918] virtqueue_add+0x493/0xc70 [97265.055925] virtqueue_add_sgs+0x80/0xa0 [97265.055929] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.055935] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.055940] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.055944] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.055948] scsi_queue_rq+0x512/0xb10 [97265.055951] __blk_mq_try_issue_directly+0x163/0x200 [97265.055957] blk_mq_request_issue_directly+0x4e/0xb0 [97265.055961] blk_mq_try_issue_list_directly+0x62/0x100 [97265.055965] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.055970] blk_mq_flush_plug_list+0x135/0x220 [97265.055975] blk_flush_plug_list+0xd7/0x100 [97265.055981] blk_finish_plug+0x25/0x36 [97265.055986] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.056023] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.056053] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.056082] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.056111] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.056141] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.056164] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.056183] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.056305] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.056419] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.056534] ? flush_work+0x42/0x1d0 [97265.056539] ? internal_add_timer+0x42/0x70 [97265.056543] ? _cond_resched+0x15/0x30 [97265.056546] ? mutex_lock+0xe/0x30 [97265.056549] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.056664] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.056789] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.056910] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.057023] ll_ost_io04_035: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97265.057033] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.057160] kthread+0x134/0x150 [97265.057164] ? set_kthread_struct+0x50/0x50 [97265.057168] ret_from_fork+0x1f/0x40 [97265.058123] ,cpuset= [97265.059072] / [97265.059072] ,mems_allowed=0 [97265.059076] CPU: 19 PID: 527695 Comm: ll_ost_io04_058 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.059080] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.059082] Call Trace: [97265.059085] dump_stack+0x41/0x60 [97265.059091] warn_alloc.cold.127+0x7b/0x108 [97265.059097] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.059100] ? __alloc_pages_nodemask+0x166/0x330 [97265.059103] __alloc_pages_nodemask+0x2e2/0x330 [97265.059107] kmalloc_order+0x28/0x90 [97265.059112] kmalloc_order_trace+0x1d/0xb0 [97265.059116] __kmalloc+0x203/0x250 [97265.059121] virtqueue_add+0x493/0xc70 [97265.059127] ? finish_wait+0x80/0x80 [97265.059131] virtqueue_add_sgs+0x80/0xa0 [97265.059136] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.059142] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.059147] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.059151] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.059155] scsi_queue_rq+0x512/0xb10 [97265.059158] __blk_mq_try_issue_directly+0x163/0x200 [97265.059165] blk_mq_request_issue_directly+0x4e/0xb0 [97265.059170] blk_mq_try_issue_list_directly+0x62/0x100 [97265.059174] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.059178] blk_mq_flush_plug_list+0x135/0x220 [97265.059183] blk_flush_plug_list+0xd7/0x100 [97265.059188] blk_finish_plug+0x25/0x36 [97265.059193] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.059231] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.059263] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.059290] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.059319] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.059348] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.059370] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.059389] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.059510] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.059631] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.059749] ? flush_work+0x42/0x1d0 [97265.059754] ? internal_add_timer+0x42/0x70 [97265.059757] ? _cond_resched+0x15/0x30 [97265.059759] ? mutex_lock+0xe/0x30 [97265.059763] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.059878] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.059993] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.060115] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.060238] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.060358] kthread+0x134/0x150 [97265.060362] ? set_kthread_struct+0x50/0x50 [97265.060365] ret_from_fork+0x1f/0x40 [97265.602938] / [97265.602937] ,mems_allowed=0 [97265.603404] ,mems_allowed=0 [97265.604279] CPU: 16 PID: 527352 Comm: ll_ost_io04_035 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.607873] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.609580] Call Trace: [97265.610328] dump_stack+0x41/0x60 [97265.611227] warn_alloc.cold.127+0x7b/0x108 [97265.612324] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.613491] ? __alloc_pages_nodemask+0x166/0x330 [97265.614682] __alloc_pages_nodemask+0x2e2/0x330 [97265.615859] kmalloc_order+0x28/0x90 [97265.616851] kmalloc_order_trace+0x1d/0xb0 [97265.617925] __kmalloc+0x203/0x250 [97265.618848] ? __wake_up_common+0x7a/0x190 [97265.619912] virtqueue_add+0x493/0xc70 [97265.620901] ? finish_wait+0x80/0x80 [97265.621897] virtqueue_add_sgs+0x80/0xa0 [97265.622937] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.624263] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.625370] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.626612] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.627979] scsi_queue_rq+0x512/0xb10 [97265.628979] __blk_mq_try_issue_directly+0x163/0x200 [97265.630208] blk_mq_request_issue_directly+0x4e/0xb0 [97265.631422] blk_mq_try_issue_list_directly+0x62/0x100 [97265.632671] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.633859] blk_mq_flush_plug_list+0x135/0x220 [97265.635011] blk_flush_plug_list+0xd7/0x100 [97265.636083] blk_finish_plug+0x25/0x36 [97265.637073] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.638475] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [97265.639771] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.641290] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.642807] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.644148] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.645383] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.646498] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.647797] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.648998] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.650190] ? flush_work+0x42/0x1d0 [97265.650941] ? internal_add_timer+0x42/0x70 [97265.651743] ? _cond_resched+0x15/0x30 [97265.652455] ? mutex_lock+0xe/0x30 [97265.653164] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.654279] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.655658] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.657183] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.658446] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.659788] kthread+0x134/0x150 [97265.660656] ? set_kthread_struct+0x50/0x50 [97265.661723] ret_from_fork+0x1f/0x40 [97265.662680] CPU: 0 PID: 527433 Comm: ll_ost_io00_056 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97265.665163] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97265.667111] Call Trace: [97265.667823] dump_stack+0x41/0x60 [97265.668715] warn_alloc.cold.127+0x7b/0x108 [97265.669783] __alloc_pages_slowpath+0xcb2/0xcd0 [97265.670905] ? __alloc_pages_nodemask+0x166/0x330 [97265.672044] __alloc_pages_nodemask+0x2e2/0x330 [97265.673147] kmalloc_order+0x28/0x90 [97265.674072] kmalloc_order_trace+0x1d/0xb0 [97265.675092] __kmalloc+0x203/0x250 [97265.675984] ? __wake_up_common+0x7a/0x190 [97265.677006] virtqueue_add+0x493/0xc70 [97265.677954] ? finish_wait+0x80/0x80 [97265.678696] virtqueue_add_sgs+0x80/0xa0 [97265.679411] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97265.680327] ? scsi_alloc_sgtables+0x84/0x1c0 [97265.681102] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97265.682006] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97265.683017] scsi_queue_rq+0x512/0xb10 [97265.683907] __blk_mq_try_issue_directly+0x163/0x200 [97265.685076] blk_mq_request_issue_directly+0x4e/0xb0 [97265.686250] blk_mq_try_issue_list_directly+0x62/0x100 [97265.687471] blk_mq_sched_insert_requests+0xa4/0xf0 [97265.688634] blk_mq_flush_plug_list+0x135/0x220 [97265.689734] blk_flush_plug_list+0xd7/0x100 [97265.690759] blk_finish_plug+0x25/0x36 [97265.691715] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97265.693094] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97265.694363] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.695848] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97265.697292] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97265.698562] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97265.699755] ofd_commitrw+0x5b4/0xd20 [ofd] [97265.700808] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.702062] obd_commitrw+0x1b6/0x370 [ptlrpc] [97265.703283] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97265.704549] ? flush_work+0x42/0x1d0 [97265.705497] ? internal_add_timer+0x42/0x70 [97265.706587] ? _cond_resched+0x15/0x30 [97265.707558] ? mutex_lock+0xe/0x30 [97265.708459] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97265.709815] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97265.711291] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97265.712784] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97265.714009] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97265.715379] kthread+0x134/0x150 [97265.716262] ? set_kthread_struct+0x50/0x50 [97265.717307] ret_from_fork+0x1f/0x40 [97272.965592] warn_alloc: 2 callbacks suppressed [97272.965597] ll_ost_io04_055: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97272.969992] CPU: 16 PID: 527692 Comm: ll_ost_io04_055 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97272.970908] ll_ost_io02_060: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97272.972948] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97272.972950] /,mems_allowed=0 [97272.975567] Call Trace: [97272.978823] dump_stack+0x41/0x60 [97272.981525] warn_alloc.cold.127+0x7b/0x108 [97272.982702] __alloc_pages_slowpath+0xcb2/0xcd0 [97272.983959] ? __alloc_pages_nodemask+0x166/0x330 [97272.985236] __alloc_pages_nodemask+0x2e2/0x330 [97272.986554] kmalloc_order+0x28/0x90 [97272.987590] kmalloc_order_trace+0x1d/0xb0 [97272.988757] __kmalloc+0x203/0x250 [97272.989799] ? __wake_up_common+0x7a/0x190 [97272.990967] virtqueue_add+0x493/0xc70 [97272.992070] ? finish_wait+0x80/0x80 [97272.993088] virtqueue_add_sgs+0x80/0xa0 [97272.994194] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97272.995581] ? scsi_alloc_sgtables+0x84/0x1c0 [97272.996777] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97272.998117] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97272.999592] scsi_queue_rq+0x512/0xb10 [97273.000718] __blk_mq_try_issue_directly+0x163/0x200 [97273.002057] blk_mq_request_issue_directly+0x4e/0xb0 [97273.003389] blk_mq_try_issue_list_directly+0x62/0x100 [97273.004751] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.006059] blk_mq_flush_plug_list+0x135/0x220 [97273.007303] blk_flush_plug_list+0xd7/0x100 [97273.008445] blk_finish_plug+0x25/0x36 [97273.009497] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.010993] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.011165] ll_ost_io00_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97273.011186] ll_ost_io02_036: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97273.011206] ll_ost_io00_016: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97273.011215] ll_ost_io00_033: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97273.011221] ll_ost_io00_025: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97273.011225] ,cpuset= [97273.011226] / [97273.011227] / [97273.011227] ,mems_allowed=0 [97273.011228] ,mems_allowed=0 [97273.012389] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.014155] ll_ost_io02_053: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97273.014405] ,cpuset= [97273.017272] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.019573] / [97273.020766] ll_ost_io04_058: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97273.022013] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.024085] ,mems_allowed=0 [97273.024086] /,mems_allowed=0 [97273.024932] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.026215] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.041891] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.042115] ll_ost_io03_003: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97273.043048] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.045893] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.046823] ? flush_work+0x42/0x1d0 [97273.047511] ? internal_add_timer+0x42/0x70 [97273.048279] ? _cond_resched+0x15/0x30 [97273.048979] ? mutex_lock+0xe/0x30 [97273.049636] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.050621] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.051685] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.052765] ? finish_wait+0x80/0x80 [97273.053478] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.054382] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.055390] kthread+0x134/0x150 [97273.056055] ? set_kthread_struct+0x50/0x50 [97273.056829] ret_from_fork+0x1f/0x40 [97273.057516] warn_alloc_show_mem: 9 callbacks suppressed [97273.057517] CPU: 0 PID: 527271 Comm: ll_ost_io00_033 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.057519] Mem-Info: [97273.058434] active_anon:27954 inactive_anon:200001 isolated_anon:0 active_file:31804070 inactive_file:2821538 isolated_file:1 unevictable:0 dirty:1104 writeback:0 slab_reclaimable:1000966 slab_unreclaimable:605022 mapped:23054 shmem:146 pagetables:5697 bounce:0 free:1552208 free_pcp:491 free_cma:0 [97273.060763] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.061319] Node 0 active_anon:111816kB inactive_anon:800004kB active_file:127216280kB inactive_file:11286152kB unevictable:0kB isolated(anon):0kB isolated(file):4kB mapped:92216kB dirty:4416kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35376kB pagetables:22788kB all_unreclaimable? no [97273.067938] Call Trace: [97273.069504] Node 0 [97273.075035] dump_stack+0x41/0x60 [97273.075615] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97273.076242] warn_alloc.cold.127+0x7b/0x108 [97273.076942] lowmem_reserve[]: [97273.080914] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.081757] 0 [97273.082555] ? __alloc_pages_nodemask+0x166/0x330 [97273.083429] 913 [97273.084009] __alloc_pages_nodemask+0x2e2/0x330 [97273.084897] 150007 [97273.085492] kmalloc_order+0x28/0x90 [97273.086380] 150007 [97273.087022] kmalloc_order_trace+0x1d/0xb0 [97273.087814] 150007 [97273.088444] __kmalloc+0x203/0x250 [97273.089939] virtqueue_add+0x493/0xc70 [97273.090680] Node 0 [97273.091236] ? finish_wait+0x80/0x80 [97273.092054] DMA32 free:596496kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97273.092713] virtqueue_add_sgs+0x80/0xa0 [97273.093486] lowmem_reserve[]: [97273.097730] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.098533] 0 [97273.099322] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.100316] 0 [97273.100909] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.101762] 149093 [97273.102330] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.103560] 149093 [97273.104213] scsi_queue_rq+0x512/0xb10 [97273.105268] 149093 [97273.105947] __blk_mq_try_issue_directly+0x163/0x200 [97273.107413] blk_mq_request_issue_directly+0x4e/0xb0 [97273.108357] Node 0 [97273.108922] blk_mq_try_issue_list_directly+0x62/0x100 [97273.109857] Normal free:5555804kB min:49268kB low:201940kB high:354612kB active_anon:111436kB inactive_anon:800000kB active_file:127265508kB inactive_file:11283412kB unevictable:0kB writepending:4416kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1448kB local_pcp:108kB free_cma:0kB [97273.110512] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.111465] lowmem_reserve[]: [97273.116280] blk_mq_flush_plug_list+0x135/0x220 [97273.117267] 0 [97273.118086] blk_flush_plug_list+0xd7/0x100 [97273.119006] 0 [97273.119619] blk_finish_plug+0x25/0x36 [97273.120467] 0 [97273.121056] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.121829] 0 [97273.122404] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.123466] 0 [97273.124046] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.125599] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.126714] Node 0 [97273.127265] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.128363] DMA: [97273.129001] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.129990] 0*4kB [97273.130590] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.131506] 0*8kB [97273.132126] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.132906] 0*16kB [97273.133521] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.134368] 0*32kB [97273.134995] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.135843] 0*64kB [97273.136484] ? flush_work+0x42/0x1d0 [97273.137380] 0*128kB [97273.138017] ? internal_add_timer+0x42/0x70 [97273.138742] 0*256kB [97273.139370] ? _cond_resched+0x15/0x30 [97273.140172] 0*512kB [97273.140802] ? mutex_lock+0xe/0x30 [97273.141563] 1*1024kB [97273.142199] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.142887] (U) [97273.143547] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.144457] 1*2048kB [97273.145041] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.146043] (M) [97273.146716] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.147724] 2*4096kB [97273.148313] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.149138] (M) [97273.149783] kthread+0x134/0x150 [97273.150704] = 11264kB [97273.151288] ? set_kthread_struct+0x50/0x50 [97273.151966] Node 0 [97273.152603] ret_from_fork+0x1f/0x40 [97273.153392] DMA32: [97273.153998] CPU: 10 PID: 527317 Comm: ll_ost_io02_053 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.154733] 18*4kB [97273.155373] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.157435] (UM) [97273.158165] Call Trace: [97273.159710] 21*8kB [97273.160323] dump_stack+0x41/0x60 [97273.160911] (UMH) [97273.161556] warn_alloc.cold.127+0x7b/0x108 [97273.162265] 28*16kB [97273.162889] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.163702] (UME) [97273.164353] ? __alloc_pages_nodemask+0x166/0x330 [97273.165217] 45*32kB [97273.165825] __alloc_pages_nodemask+0x2e2/0x330 [97273.166712] (UMH) [97273.167337] kmalloc_order+0x28/0x90 [97273.168214] 44*64kB [97273.168785] kmalloc_order_trace+0x1d/0xb0 [97273.169665] (UM) [97273.170292] __kmalloc+0x203/0x250 [97273.171111] 46*128kB [97273.171691] ? __wake_up_common+0x7a/0x190 [97273.172397] (UM) [97273.173046] virtqueue_add+0x493/0xc70 [97273.173838] 44*256kB [97273.174433] ? finish_wait+0x80/0x80 [97273.175183] (UM) [97273.175835] virtqueue_add_sgs+0x80/0xa0 [97273.176558] 46*512kB [97273.177167] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.177943] (UMEH) [97273.178613] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.179603] 38*1024kB [97273.180240] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.181083] (UM) [97273.181758] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.182690] 42*2048kB [97273.183315] scsi_queue_rq+0x512/0xb10 [97273.184339] (UM) [97273.185018] __blk_mq_try_issue_directly+0x163/0x200 [97273.185777] 104*4096kB [97273.186384] blk_mq_request_issue_directly+0x4e/0xb0 [97273.187379] (UM) [97273.188059] blk_mq_try_issue_list_directly+0x62/0x100 [97273.188993] = 596560kB [97273.189604] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.190558] Node 0 [97273.191240] blk_mq_flush_plug_list+0x135/0x220 [97273.192158] Normal: [97273.192809] blk_flush_plug_list+0xd7/0x100 [97273.193676] 339586*4kB [97273.194331] blk_finish_plug+0x25/0x36 [97273.195161] (UME) [97273.195843] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.196588] 478586*8kB [97273.197201] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.198210] (UME) [97273.198900] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.199834] 20681*16kB [97273.200449] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.201547] (UMEH) [97273.202218] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.203286] 20*32kB [97273.203916] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.204871] (UH) [97273.205520] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.206484] 1*64kB [97273.207065] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.207846] (H) [97273.208479] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.209307] 0*128kB [97273.209881] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.210687] 0*256kB [97273.211323] ? flush_work+0x42/0x1d0 [97273.212192] 0*512kB [97273.212841] ? internal_add_timer+0x42/0x70 [97273.213553] 0*1024kB [97273.214194] ? _cond_resched+0x15/0x30 [97273.215009] 0*2048kB [97273.215673] ? mutex_lock+0xe/0x30 [97273.216449] 0*4096kB [97273.217121] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.217791] = 5518632kB [97273.218457] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.219380] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97273.220070] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.221080] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97273.222799] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.223817] 34648549 total pagecache pages [97273.225503] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.226324] 1607 pages in swap cache [97273.227289] kthread+0x134/0x150 [97273.228229] Swap cache stats: add 1607363, delete 1605779, find 83387/100594 [97273.229114] ? set_kthread_struct+0x50/0x50 [97273.229800] Free swap = 5333060kB [97273.231256] ret_from_fork+0x1f/0x40 [97273.232096] Total swap = 11075580kB [97273.232965] CPU: 2 PID: 527182 Comm: ll_ost_io00_016 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.233785] 39321462 pages RAM [97273.234603] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.237524] 0 pages HighMem/MovableOnly [97273.238293] Call Trace: [97273.240413] 905848 pages reserved [97273.241319] dump_stack+0x41/0x60 [97273.242176] 0 pages hwpoisoned [97273.243000] warn_alloc.cold.127+0x7b/0x108 [97273.245679] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.246657] ? __alloc_pages_nodemask+0x166/0x330 [97273.247656] __alloc_pages_nodemask+0x2e2/0x330 [97273.248605] kmalloc_order+0x28/0x90 [97273.249403] kmalloc_order_trace+0x1d/0xb0 [97273.250279] __kmalloc+0x203/0x250 [97273.251037] ? __wake_up_common+0x7a/0x190 [97273.251905] virtqueue_add+0x493/0xc70 [97273.252725] ? finish_wait+0x80/0x80 [97273.253499] virtqueue_add_sgs+0x80/0xa0 [97273.254367] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.255455] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.256368] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.257390] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.258507] scsi_queue_rq+0x512/0xb10 [97273.259332] __blk_mq_try_issue_directly+0x163/0x200 [97273.260350] blk_mq_request_issue_directly+0x4e/0xb0 [97273.261369] blk_mq_try_issue_list_directly+0x62/0x100 [97273.262415] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.263421] blk_mq_flush_plug_list+0x135/0x220 [97273.264382] blk_flush_plug_list+0xd7/0x100 [97273.265305] blk_finish_plug+0x25/0x36 [97273.266162] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.267363] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.268470] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.269741] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.270972] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.272062] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.273074] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.274004] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.275101] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.276151] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.277235] ? flush_work+0x42/0x1d0 [97273.278022] ? internal_add_timer+0x42/0x70 [97273.278904] ? _cond_resched+0x15/0x30 [97273.279729] ? mutex_lock+0xe/0x30 [97273.280488] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.281617] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.282859] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.284115] ? finish_wait+0x80/0x80 [97273.284904] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.285931] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.287036] kthread+0x134/0x150 [97273.287749] ? set_kthread_struct+0x50/0x50 [97273.288612] ret_from_fork+0x1f/0x40 [97273.289380] CPU: 18 PID: 527695 Comm: ll_ost_io04_058 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.293049] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.295183] Call Trace: [97273.295948] dump_stack+0x41/0x60 [97273.296886] warn_alloc.cold.127+0x7b/0x108 [97273.297997] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.299025] ? __alloc_pages_nodemask+0x166/0x330 [97273.300243] __alloc_pages_nodemask+0x2e2/0x330 [97273.301424] kmalloc_order+0x28/0x90 [97273.302436] kmalloc_order_trace+0x1d/0xb0 [97273.303552] __kmalloc+0x203/0x250 [97273.304560] ? mempool_alloc+0x67/0x180 [97273.305681] virtqueue_add+0x493/0xc70 [97273.306778] virtqueue_add_sgs+0x80/0xa0 [97273.307868] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.309234] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.310396] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.311677] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.313088] scsi_queue_rq+0x512/0xb10 [97273.313857] __blk_mq_try_issue_directly+0x163/0x200 [97273.315053] blk_mq_request_issue_directly+0x4e/0xb0 [97273.316328] blk_mq_try_issue_list_directly+0x62/0x100 [97273.317633] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.318839] blk_mq_flush_plug_list+0x135/0x220 [97273.319919] blk_flush_plug_list+0xd7/0x100 [97273.321037] blk_finish_plug+0x25/0x36 [97273.322063] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.323590] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.325197] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.326755] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.328070] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.329189] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.330322] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.331681] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.332994] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.334341] ? req_can_reconstruct+0x32/0xf0 [ptlrpc] [97273.335803] ? _cond_resched+0x15/0x30 [97273.336799] ? mutex_lock+0xe/0x30 [97273.337720] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.339133] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.340695] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.342296] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.343379] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.344737] kthread+0x134/0x150 [97273.345645] ? set_kthread_struct+0x50/0x50 [97273.346705] ret_from_fork+0x1f/0x40 [97273.347633] CPU: 8 PID: 527335 Comm: ll_ost_io02_060 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.349705] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.351210] Call Trace: [97273.351743] dump_stack+0x41/0x60 [97273.352399] warn_alloc.cold.127+0x7b/0x108 [97273.353175] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.353994] ? __alloc_pages_nodemask+0x166/0x330 [97273.354851] __alloc_pages_nodemask+0x2e2/0x330 [97273.355674] kmalloc_order+0x28/0x90 [97273.356361] kmalloc_order_trace+0x1d/0xb0 [97273.357139] __kmalloc+0x203/0x250 [97273.357793] virtqueue_add+0x493/0xc70 [97273.358506] virtqueue_add_sgs+0x80/0xa0 [97273.359239] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.360188] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.361003] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.361901] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.362881] scsi_queue_rq+0x512/0xb10 [97273.363586] __blk_mq_try_issue_directly+0x163/0x200 [97273.364506] blk_mq_request_issue_directly+0x4e/0xb0 [97273.365388] blk_mq_try_issue_list_directly+0x62/0x100 [97273.366292] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.367171] blk_mq_flush_plug_list+0x135/0x220 [97273.367985] blk_flush_plug_list+0xd7/0x100 [97273.368750] blk_finish_plug+0x25/0x36 [97273.369452] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.370480] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.371417] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.372525] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.373605] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.374554] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.375434] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.376220] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.377151] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.378047] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.378967] ? flush_work+0x42/0x1d0 [97273.379659] ? internal_add_timer+0x42/0x70 [97273.380426] ? _cond_resched+0x15/0x30 [97273.381129] ? mutex_lock+0xe/0x30 [97273.381781] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.382731] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.383781] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.384849] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.385724] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.386673] kthread+0x134/0x150 [97273.387309] ? set_kthread_struct+0x50/0x50 [97273.388093] ret_from_fork+0x1f/0x40 [97273.388807] CPU: 12 PID: 527010 Comm: ll_ost_io03_003 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.391289] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.392783] Call Trace: [97273.393326] dump_stack+0x41/0x60 [97273.393968] warn_alloc.cold.127+0x7b/0x108 [97273.394744] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.395586] __alloc_pages_nodemask+0x2e2/0x330 [97273.396412] kmalloc_order+0x28/0x90 [97273.397130] kmalloc_order_trace+0x1d/0xb0 [97273.397912] __kmalloc+0x203/0x250 [97273.398586] virtqueue_add+0x493/0xc70 [97273.399370] ? finish_wait+0x80/0x80 [97273.400096] virtqueue_add_sgs+0x80/0xa0 [97273.400890] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.401851] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.402655] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.403563] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.404592] scsi_queue_rq+0x512/0xb10 [97273.405378] __blk_mq_try_issue_directly+0x163/0x200 [97273.406273] blk_mq_request_issue_directly+0x4e/0xb0 [97273.407168] blk_mq_try_issue_list_directly+0x62/0x100 [97273.408106] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.408985] blk_mq_flush_plug_list+0x135/0x220 [97273.409813] blk_flush_plug_list+0xd7/0x100 [97273.410611] blk_finish_plug+0x25/0x36 [97273.411334] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.412362] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.413361] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.414485] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.415590] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.416555] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.417485] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.418274] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.419214] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.420190] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.421162] ? set_next_entity+0xb5/0x1e0 [97273.421902] ? internal_add_timer+0x42/0x70 [97273.422714] ? _cond_resched+0x15/0x30 [97273.423458] ? mutex_lock+0xe/0x30 [97273.424158] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.425196] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.426312] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.427410] ? finish_wait+0x80/0x80 [97273.428119] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.429027] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.430011] kthread+0x134/0x150 [97273.430685] ? set_kthread_struct+0x50/0x50 [97273.431456] ret_from_fork+0x1f/0x40 [97273.432142] CPU: 1 PID: 527214 Comm: ll_ost_io00_025 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.434727] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.436621] Call Trace: [97273.437263] dump_stack+0x41/0x60 [97273.438041] warn_alloc.cold.127+0x7b/0x108 [97273.439023] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.440029] ? __alloc_pages_nodemask+0x166/0x330 [97273.440997] __alloc_pages_nodemask+0x2e2/0x330 [97273.441939] kmalloc_order+0x28/0x90 [97273.442727] kmalloc_order_trace+0x1d/0xb0 [97273.443630] __kmalloc+0x203/0x250 [97273.444402] virtqueue_add+0x493/0xc70 [97273.445229] virtqueue_add_sgs+0x80/0xa0 [97273.446105] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.447219] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.448153] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.449214] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.450363] scsi_queue_rq+0x512/0xb10 [97273.451194] __blk_mq_try_issue_directly+0x163/0x200 [97273.452224] blk_mq_request_issue_directly+0x4e/0xb0 [97273.453249] blk_mq_try_issue_list_directly+0x62/0x100 [97273.454305] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.455317] blk_mq_flush_plug_list+0x135/0x220 [97273.456274] blk_flush_plug_list+0xd7/0x100 [97273.457175] blk_finish_plug+0x25/0x36 [97273.457990] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.459189] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.460293] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.461567] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.462817] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.463937] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.464968] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.465888] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.466959] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.467989] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.469099] ? flush_work+0x42/0x1d0 [97273.469902] ? internal_add_timer+0x42/0x70 [97273.470797] ? _cond_resched+0x15/0x30 [97273.471619] ? mutex_lock+0xe/0x30 [97273.472380] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.473521] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.474758] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.476012] ? finish_wait+0x80/0x80 [97273.476805] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.477818] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.478922] kthread+0x134/0x150 [97273.479650] ? set_kthread_struct+0x50/0x50 [97273.480541] ret_from_fork+0x1f/0x40 [97273.481340] CPU: 3 PID: 527758 Comm: ll_ost_io00_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.484121] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.486186] Call Trace: [97273.486938] dump_stack+0x41/0x60 [97273.487842] warn_alloc.cold.127+0x7b/0x108 [97273.488904] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.490021] ? __alloc_pages_nodemask+0x166/0x330 [97273.491181] __alloc_pages_nodemask+0x2e2/0x330 [97273.492315] kmalloc_order+0x28/0x90 [97273.493275] kmalloc_order_trace+0x1d/0xb0 [97273.494326] __kmalloc+0x203/0x250 [97273.495240] virtqueue_add+0x493/0xc70 [97273.496227] ? finish_wait+0x80/0x80 [97273.497180] virtqueue_add_sgs+0x80/0xa0 [97273.498189] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.499479] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.500661] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.501917] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.503311] scsi_queue_rq+0x512/0xb10 [97273.504350] __blk_mq_try_issue_directly+0x163/0x200 [97273.505591] blk_mq_request_issue_directly+0x4e/0xb0 [97273.506825] blk_mq_try_issue_list_directly+0x62/0x100 [97273.508126] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.509353] blk_mq_flush_plug_list+0x135/0x220 [97273.510546] blk_flush_plug_list+0xd7/0x100 [97273.511580] blk_finish_plug+0x25/0x36 [97273.512557] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.513975] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.515475] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.516971] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.518278] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.519486] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.520587] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.521859] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.523061] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.524306] ? flush_work+0x42/0x1d0 [97273.525294] ? internal_add_timer+0x42/0x70 [97273.526373] ? _cond_resched+0x15/0x30 [97273.527346] ? mutex_lock+0xe/0x30 [97273.528263] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.529678] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.531186] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.532745] ? finish_wait+0x80/0x80 [97273.533731] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.534967] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.536301] kthread+0x134/0x150 [97273.537214] ? set_kthread_struct+0x50/0x50 [97273.538109] ret_from_fork+0x1f/0x40 [97273.538811] CPU: 11 PID: 527212 Comm: ll_ost_io02_036 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97273.541660] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97273.543746] Call Trace: [97273.544509] dump_stack+0x41/0x60 [97273.545419] warn_alloc.cold.127+0x7b/0x108 [97273.546530] __alloc_pages_slowpath+0xcb2/0xcd0 [97273.547691] ? __alloc_pages_nodemask+0x166/0x330 [97273.548885] __alloc_pages_nodemask+0x2e2/0x330 [97273.550060] kmalloc_order+0x28/0x90 [97273.551019] kmalloc_order_trace+0x1d/0xb0 [97273.552100] __kmalloc+0x203/0x250 [97273.553020] virtqueue_add+0x493/0xc70 [97273.554005] ? finish_wait+0x80/0x80 [97273.554947] virtqueue_add_sgs+0x80/0xa0 [97273.555971] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97273.557304] ? scsi_alloc_sgtables+0x84/0x1c0 [97273.558439] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97273.559532] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97273.560763] scsi_queue_rq+0x512/0xb10 [97273.561856] __blk_mq_try_issue_directly+0x163/0x200 [97273.562973] blk_mq_request_issue_directly+0x4e/0xb0 [97273.563883] blk_mq_try_issue_list_directly+0x62/0x100 [97273.564815] blk_mq_sched_insert_requests+0xa4/0xf0 [97273.565690] blk_mq_flush_plug_list+0x135/0x220 [97273.566515] blk_flush_plug_list+0xd7/0x100 [97273.567290] blk_finish_plug+0x25/0x36 [97273.567993] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97273.569025] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97273.569983] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.571133] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97273.572233] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97273.573190] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97273.574063] ofd_commitrw+0x5b4/0xd20 [ofd] [97273.574855] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.575791] obd_commitrw+0x1b6/0x370 [ptlrpc] [97273.576681] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97273.577610] ? flush_work+0x42/0x1d0 [97273.578302] ? internal_add_timer+0x42/0x70 [97273.579056] ? _cond_resched+0x15/0x30 [97273.579781] ? mutex_lock+0xe/0x30 [97273.580460] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97273.581456] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97273.582547] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97273.583613] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97273.584505] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97273.585462] kthread+0x134/0x150 [97273.586115] ? set_kthread_struct+0x50/0x50 [97273.586890] ret_from_fork+0x1f/0x40 [97337.737883] warn_alloc: 1 callbacks suppressed [97337.737887] ll_ost_io01_074: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97337.739947] ll_ost_io02_074: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97337.742609] CPU: 7 PID: 528143 Comm: ll_ost_io01_074 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97337.744816] ll_ost_io04_021: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97337.751070] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97337.753184] Call Trace: [97337.754003] dump_stack+0x41/0x60 [97337.754981] warn_alloc.cold.127+0x7b/0x108 [97337.756132] __alloc_pages_slowpath+0xcb2/0xcd0 [97337.757133] ll_ost_io00_080: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97337.757323] ? __alloc_pages_nodemask+0x166/0x330 [97337.761344] __alloc_pages_nodemask+0x2e2/0x330 [97337.762564] kmalloc_order+0x28/0x90 [97337.763143] ll_ost_io01_043: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97337.763576] kmalloc_order_trace+0x1d/0xb0 [97337.765810] ll_ost_io04_071: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97337.767586] ll_ost_io01_005: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97337.769672] __kmalloc+0x203/0x250 [97337.772868] ? __wake_up_common+0x7a/0x190 [97337.773851] virtqueue_add+0x493/0xc70 [97337.774766] ? finish_wait+0x80/0x80 [97337.775604] virtqueue_add_sgs+0x80/0xa0 [97337.776510] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97337.777670] ? scsi_alloc_sgtables+0x84/0x1c0 [97337.778636] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97337.779718] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97337.780893] scsi_queue_rq+0x512/0xb10 [97337.781783] __blk_mq_try_issue_directly+0x163/0x200 [97337.782832] blk_mq_request_issue_directly+0x4e/0xb0 [97337.783897] blk_mq_try_issue_list_directly+0x62/0x100 [97337.784974] blk_mq_sched_insert_requests+0xa4/0xf0 [97337.786016] blk_mq_flush_plug_list+0x135/0x220 [97337.787007] blk_flush_plug_list+0xd7/0x100 [97337.787940] blk_finish_plug+0x25/0x36 [97337.788801] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97337.790088] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97337.791268] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97337.792577] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97337.793834] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97337.794935] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97337.795972] ofd_commitrw+0x5b4/0xd20 [ofd] [97337.796941] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97337.798080] obd_commitrw+0x1b6/0x370 [ptlrpc] [97337.799189] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97337.800283] ? flush_work+0x42/0x1d0 [97337.801100] ? internal_add_timer+0x42/0x70 [97337.802014] ? _cond_resched+0x15/0x30 [97337.802892] ? mutex_lock+0xe/0x30 [97337.803705] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97337.804863] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97337.806137] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97337.807425] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97337.808476] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97337.809616] kthread+0x134/0x150 [97337.810363] ? set_kthread_struct+0x50/0x50 [97337.811252] ret_from_fork+0x1f/0x40 [97337.812054] warn_alloc_show_mem: 9 callbacks suppressed [97337.812055] CPU: 6 PID: 527012 Comm: ll_ost_io01_005 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97337.812063] Mem-Info: [97337.813147] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97337.815491] active_anon:27953 inactive_anon:200627 isolated_anon:0 active_file:31973881 inactive_file:3155906 isolated_file:22 unevictable:0 dirty:1186 writeback:0 slab_reclaimable:1000936 slab_unreclaimable:605495 mapped:23409 shmem:146 pagetables:5812 bounce:0 free:1046979 free_pcp:488 free_cma:0 [97337.816120] Call Trace: [97337.817846] Node 0 active_anon:111812kB inactive_anon:802508kB active_file:127907624kB inactive_file:12623624kB unevictable:0kB isolated(anon):0kB isolated(file):88kB mapped:93636kB dirty:4744kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35504kB pagetables:23248kB all_unreclaimable? no [97337.824539] dump_stack+0x41/0x60 [97337.825170] Node 0 [97337.830725] warn_alloc.cold.127+0x7b/0x108 [97337.831517] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97337.832111] __alloc_pages_slowpath+0xcb2/0xcd0 [97337.833029] lowmem_reserve[]: [97337.837018] ? __alloc_pages_nodemask+0x166/0x330 [97337.838026] 0 [97337.838785] __alloc_pages_nodemask+0x2e2/0x330 [97337.839809] 913 [97337.840343] kmalloc_order+0x28/0x90 [97337.841339] 150007 [97337.841902] kmalloc_order_trace+0x1d/0xb0 [97337.842733] 150007 [97337.843330] __kmalloc+0x203/0x250 [97337.844248] 150007 [97337.844843] virtqueue_add+0x493/0xc70 [97337.846272] ? finish_wait+0x80/0x80 [97337.847141] Node 0 [97337.847651] virtqueue_add_sgs+0x80/0xa0 [97337.848483] DMA32 free:596640kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97337.849087] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97337.849997] lowmem_reserve[]: [97337.854246] ? scsi_alloc_sgtables+0x84/0x1c0 [97337.855375] 0 [97337.856152] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97337.857138] 0 [97337.857700] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97337.858782] 149093 [97337.859332] scsi_queue_rq+0x512/0xb10 [97337.860529] 149093 [97337.861164] __blk_mq_try_issue_directly+0x163/0x200 [97337.862043] 149093 [97337.862679] blk_mq_request_issue_directly+0x4e/0xb0 [97337.864420] blk_mq_try_issue_list_directly+0x62/0x100 [97337.865526] Node 0 [97337.866074] blk_mq_sched_insert_requests+0xa4/0xf0 [97337.867186] Normal free:3559604kB min:49268kB low:201940kB high:354612kB active_anon:111432kB inactive_anon:802464kB active_file:127924424kB inactive_file:12614324kB unevictable:0kB writepending:4744kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1948kB local_pcp:144kB free_cma:0kB [97337.867804] blk_mq_flush_plug_list+0x135/0x220 [97337.868863] lowmem_reserve[]: [97337.873741] blk_flush_plug_list+0xd7/0x100 [97337.874774] 0 [97337.875557] blk_finish_plug+0x25/0x36 [97337.876530] 0 [97337.877086] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97337.877951] 0 [97337.878487] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97337.879679] 0 [97337.880205] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97337.881293] 0 [97337.881812] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97337.883580] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97337.884815] Node 0 [97337.885304] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97337.886412] DMA: [97337.887025] ofd_commitrw+0x5b4/0xd20 [ofd] [97337.888052] 0*4kB [97337.888622] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97337.889599] 0*8kB [97337.890223] obd_commitrw+0x1b6/0x370 [ptlrpc] [97337.891266] 0*16kB [97337.891865] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97337.892862] 0*32kB [97337.893447] ? flush_work+0x42/0x1d0 [97337.894449] 0*64kB [97337.895035] ? internal_add_timer+0x42/0x70 [97337.895854] 0*128kB [97337.896432] ? _cond_resched+0x15/0x30 [97337.897352] 0*256kB [97337.897956] ? mutex_lock+0xe/0x30 [97337.898785] 0*512kB [97337.899373] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97337.900167] 1*1024kB [97337.900749] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97337.901801] (U) [97337.902418] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97337.903608] 1*2048kB [97337.904192] ? finish_wait+0x80/0x80 [97337.905362] (M) [97337.905968] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97337.906794] 2*4096kB [97337.907325] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97337.908267] (M) [97337.908861] kthread+0x134/0x150 [97337.909887] = 11264kB [97337.910405] ? set_kthread_struct+0x50/0x50 [97337.911137] Node 0 [97337.911718] ret_from_fork+0x1f/0x40 [97337.912587] DMA32: [97337.913125] CPU: 4 PID: 527147 Comm: ll_ost_io01_043 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97337.914066] 18*4kB [97337.914710] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97337.917649] (UM) [97337.918432] Call Trace: [97337.920507] 21*8kB [97337.921049] dump_stack+0x41/0x60 [97337.921852] (UM) [97337.922395] warn_alloc.cold.127+0x7b/0x108 [97337.923349] 29*16kB [97337.923838] __alloc_pages_slowpath+0xcb2/0xcd0 [97337.924822] (UME) [97337.925346] ? __alloc_pages_nodemask+0x166/0x330 [97337.926503] 48*32kB [97337.927004] __alloc_pages_nodemask+0x2e2/0x330 [97337.928212] (UMH) [97337.928763] kmalloc_order+0x28/0x90 [97337.929964] 46*64kB [97337.930477] kmalloc_order_trace+0x1d/0xb0 [97337.931485] (UMH) [97337.932020] __kmalloc+0x203/0x250 [97337.933111] 46*128kB [97337.933615] ? __wake_up_common+0x7a/0x190 [97337.934568] (UM) [97337.935118] virtqueue_add+0x493/0xc70 [97337.935953] 44*256kB [97337.936461] ? finish_wait+0x80/0x80 [97337.937230] (UM) [97337.937788] virtqueue_add_sgs+0x80/0xa0 [97337.938528] 46*512kB [97337.939048] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97337.939846] (UMEH) [97337.940422] ? scsi_alloc_sgtables+0x84/0x1c0 [97337.941410] 38*1024kB [97337.941946] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97337.942787] (UM) [97337.943366] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97337.944300] 42*2048kB [97337.944812] scsi_queue_rq+0x512/0xb10 [97337.945834] (UM) [97337.946419] __blk_mq_try_issue_directly+0x163/0x200 [97337.947181] 104*4096kB [97337.947706] blk_mq_request_issue_directly+0x4e/0xb0 [97337.948636] (UM) [97337.949225] blk_mq_try_issue_list_directly+0x62/0x100 [97337.950162] = 596800kB [97337.950676] blk_mq_sched_insert_requests+0xa4/0xf0 [97337.951630] Node 0 [97337.952199] blk_mq_flush_plug_list+0x135/0x220 [97337.953115] Normal: [97337.953658] blk_flush_plug_list+0xd7/0x100 [97337.954528] 648666*4kB [97337.955073] blk_finish_plug+0x25/0x36 [97337.955887] (UME) [97337.956477] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97337.957238] 113140*8kB [97337.957753] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97337.958787] (UME) [97337.959394] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97337.960528] 2076*16kB [97337.961056] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97337.962135] (UMEH) [97337.962701] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97337.963643] 4*32kB [97337.964182] ofd_commitrw+0x5b4/0xd20 [ofd] [97337.965190] (UH) [97337.965720] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97337.966522] 1*64kB [97337.967043] obd_commitrw+0x1b6/0x370 [ptlrpc] [97337.967908] (H) [97337.968448] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97337.969274] 0*128kB [97337.969774] ? flush_work+0x42/0x1d0 [97337.970654] 0*256kB [97337.971198] ? internal_add_timer+0x42/0x70 [97337.971931] 0*512kB [97337.972466] ? _cond_resched+0x15/0x30 [97337.973294] 0*1024kB [97337.973830] ? mutex_lock+0xe/0x30 [97337.974601] 0*2048kB [97337.975171] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97337.975850] 0*4096kB [97337.976413] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97337.977545] = 3533192kB [97337.978099] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97337.979479] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97337.980062] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97337.981453] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97337.982916] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97337.984038] 35145359 total pagecache pages [97337.985468] kthread+0x134/0x150 [97337.986682] 1620 pages in swap cache [97337.987502] ? set_kthread_struct+0x50/0x50 [97337.988195] Swap cache stats: add 1608418, delete 1606821, find 83808/101122 [97337.988936] ret_from_fork+0x1f/0x40 [97337.989753] Free swap = 5332284kB [97337.990992] CPU: 0 PID: 527785 Comm: ll_ost_io00_080 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97337.991674] Total swap = 11075580kB [97337.992646] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97337.994814] 39321462 pages RAM [97337.995730] Call Trace: [97337.997385] 0 pages HighMem/MovableOnly [97337.998316] dump_stack+0x41/0x60 [97337.998894] 905848 pages reserved [97337.999995] warn_alloc.cold.127+0x7b/0x108 [97338.000753] 0 pages hwpoisoned [97338.001712] __alloc_pages_slowpath+0xcb2/0xcd0 [97338.004692] ? __alloc_pages_nodemask+0x166/0x330 [97338.005927] __alloc_pages_nodemask+0x2e2/0x330 [97338.007060] kmalloc_order+0x28/0x90 [97338.007992] kmalloc_order_trace+0x1d/0xb0 [97338.009027] __kmalloc+0x203/0x250 [97338.009925] virtqueue_add+0x493/0xc70 [97338.010886] ? finish_wait+0x80/0x80 [97338.011814] virtqueue_add_sgs+0x80/0xa0 [97338.012805] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97338.014114] ? scsi_alloc_sgtables+0x84/0x1c0 [97338.015208] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97338.016167] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97338.017116] scsi_queue_rq+0x512/0xb10 [97338.017813] __blk_mq_try_issue_directly+0x163/0x200 [97338.018661] blk_mq_request_issue_directly+0x4e/0xb0 [97338.019529] blk_mq_try_issue_list_directly+0x62/0x100 [97338.020409] blk_mq_sched_insert_requests+0xa4/0xf0 [97338.021253] blk_mq_flush_plug_list+0x135/0x220 [97338.022041] blk_flush_plug_list+0xd7/0x100 [97338.022787] blk_finish_plug+0x25/0x36 [97338.023496] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97338.024515] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97338.025455] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.026526] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.027562] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97338.028469] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97338.029361] ofd_commitrw+0x5b4/0xd20 [ofd] [97338.030145] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.031039] obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.031929] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97338.032821] ? flush_work+0x42/0x1d0 [97338.033489] ? internal_add_timer+0x42/0x70 [97338.034247] ? _cond_resched+0x15/0x30 [97338.034951] ? mutex_lock+0xe/0x30 [97338.035598] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97338.036559] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97338.037602] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97338.038646] ? finish_wait+0x80/0x80 [97338.039319] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97338.040187] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97338.041118] kthread+0x134/0x150 [97338.041743] ? set_kthread_struct+0x50/0x50 [97338.042488] ret_from_fork+0x1f/0x40 [97338.043208] CPU: 19 PID: 528059 Comm: ll_ost_io04_071 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97338.046612] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97338.048663] Call Trace: [97338.049421] dump_stack+0x41/0x60 [97338.050360] warn_alloc.cold.127+0x7b/0x108 [97338.051448] __alloc_pages_slowpath+0xcb2/0xcd0 [97338.052584] __alloc_pages_nodemask+0x2e2/0x330 [97338.053755] kmalloc_order+0x28/0x90 [97338.054741] kmalloc_order_trace+0x1d/0xb0 [97338.055816] __kmalloc+0x203/0x250 [97338.056738] virtqueue_add+0x493/0xc70 [97338.057713] ? finish_wait+0x80/0x80 [97338.058657] virtqueue_add_sgs+0x80/0xa0 [97338.059671] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97338.060999] ? scsi_alloc_sgtables+0x84/0x1c0 [97338.062132] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97338.063383] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97338.064744] scsi_queue_rq+0x512/0xb10 [97338.065719] __blk_mq_try_issue_directly+0x163/0x200 [97338.066953] blk_mq_request_issue_directly+0x4e/0xb0 [97338.068212] blk_mq_try_issue_list_directly+0x62/0x100 [97338.069511] blk_mq_sched_insert_requests+0xa4/0xf0 [97338.070739] blk_mq_flush_plug_list+0x135/0x220 [97338.071909] blk_flush_plug_list+0xd7/0x100 [97338.073007] blk_finish_plug+0x25/0x36 [97338.073997] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97338.075460] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97338.076777] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.078310] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.079815] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97338.081203] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97338.082476] ofd_commitrw+0x5b4/0xd20 [ofd] [97338.083598] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.084902] obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.086159] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97338.087489] ? flush_work+0x42/0x1d0 [97338.088508] ? internal_add_timer+0x42/0x70 [97338.089590] ? _cond_resched+0x15/0x30 [97338.090608] ? mutex_lock+0xe/0x30 [97338.091548] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97338.092908] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97338.094490] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97338.096040] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97338.097314] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97338.098724] kthread+0x134/0x150 [97338.099643] ? set_kthread_struct+0x50/0x50 [97338.100770] ret_from_fork+0x1f/0x40 [97338.101794] CPU: 10 PID: 527865 Comm: ll_ost_io02_074 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97338.105811] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97338.107915] Call Trace: [97338.108570] dump_stack+0x41/0x60 [97338.109539] warn_alloc.cold.127+0x7b/0x108 [97338.110656] __alloc_pages_slowpath+0xcb2/0xcd0 [97338.111877] ? __alloc_pages_nodemask+0x166/0x330 [97338.113204] __alloc_pages_nodemask+0x2e2/0x330 [97338.114455] kmalloc_order+0x28/0x90 [97338.115451] kmalloc_order_trace+0x1d/0xb0 [97338.116559] __kmalloc+0x203/0x250 [97338.117521] virtqueue_add+0x493/0xc70 [97338.118552] ? finish_wait+0x80/0x80 [97338.119557] virtqueue_add_sgs+0x80/0xa0 [97338.120695] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97338.122069] ? scsi_alloc_sgtables+0x84/0x1c0 [97338.123238] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97338.124530] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97338.125947] scsi_queue_rq+0x512/0xb10 [97338.126967] __blk_mq_try_issue_directly+0x163/0x200 [97338.128261] blk_mq_request_issue_directly+0x4e/0xb0 [97338.129401] blk_mq_try_issue_list_directly+0x62/0x100 [97338.130730] blk_mq_sched_insert_requests+0xa4/0xf0 [97338.131998] blk_mq_flush_plug_list+0x135/0x220 [97338.133192] blk_flush_plug_list+0xd7/0x100 [97338.134336] blk_finish_plug+0x25/0x36 [97338.135379] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97338.136887] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97338.138338] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.139947] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.141469] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97338.142696] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97338.143872] ofd_commitrw+0x5b4/0xd20 [ofd] [97338.144941] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.146390] obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.147735] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97338.149100] ? flush_work+0x42/0x1d0 [97338.150098] ? internal_add_timer+0x42/0x70 [97338.151208] ? _cond_resched+0x15/0x30 [97338.152235] ? mutex_lock+0xe/0x30 [97338.153200] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97338.154616] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97338.156203] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97338.157764] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97338.159010] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97338.160191] kthread+0x134/0x150 [97338.161117] ? set_kthread_struct+0x50/0x50 [97338.162220] ret_from_fork+0x1f/0x40 [97338.163197] CPU: 17 PID: 527236 Comm: ll_ost_io04_021 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97338.166241] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97338.168415] Call Trace: [97338.169212] dump_stack+0x41/0x60 [97338.170131] warn_alloc.cold.127+0x7b/0x108 [97338.171141] __alloc_pages_slowpath+0xcb2/0xcd0 [97338.172325] __alloc_pages_nodemask+0x2e2/0x330 [97338.173498] kmalloc_order+0x28/0x90 [97338.174497] kmalloc_order_trace+0x1d/0xb0 [97338.175606] __kmalloc+0x203/0x250 [97338.176556] virtqueue_add+0x493/0xc70 [97338.177559] ? finish_wait+0x80/0x80 [97338.178401] virtqueue_add_sgs+0x80/0xa0 [97338.179254] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97338.180583] ? scsi_alloc_sgtables+0x84/0x1c0 [97338.181849] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97338.183158] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97338.184301] scsi_queue_rq+0x512/0xb10 [97338.185142] __blk_mq_try_issue_directly+0x163/0x200 [97338.186195] blk_mq_request_issue_directly+0x4e/0xb0 [97338.187387] blk_mq_try_issue_list_directly+0x62/0x100 [97338.188343] blk_mq_sched_insert_requests+0xa4/0xf0 [97338.189306] blk_mq_flush_plug_list+0x135/0x220 [97338.190376] blk_flush_plug_list+0xd7/0x100 [97338.191432] blk_finish_plug+0x25/0x36 [97338.192427] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97338.193837] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97338.195129] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.196636] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.198144] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97338.199447] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97338.200634] ofd_commitrw+0x5b4/0xd20 [ofd] [97338.201701] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.202974] obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.204258] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97338.205624] ? flush_work+0x42/0x1d0 [97338.206475] ? internal_add_timer+0x42/0x70 [97338.207545] ? _cond_resched+0x15/0x30 [97338.208312] ? mutex_lock+0xe/0x30 [97338.209013] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97338.210113] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97338.211221] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97338.212390] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97338.213549] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97338.214688] kthread+0x134/0x150 [97338.215360] ? set_kthread_struct+0x50/0x50 [97338.216141] ret_from_fork+0x1f/0x40 [97338.437288] ll_ost_io00_049: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97338.440198] CPU: 1 PID: 527426 Comm: ll_ost_io00_049 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97338.440200] ll_ost_io02_039: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97338.443100] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97338.443102] Call Trace: [97338.445018] / [97338.447102] dump_stack+0x41/0x60 [97338.447991] ,mems_allowed=0 [97338.448662] warn_alloc.cold.127+0x7b/0x108 [97338.450449] __alloc_pages_slowpath+0xcb2/0xcd0 [97338.453245] ? __alloc_pages_nodemask+0x166/0x330 [97338.454328] __alloc_pages_nodemask+0x2e2/0x330 [97338.455363] kmalloc_order+0x28/0x90 [97338.456217] kmalloc_order_trace+0x1d/0xb0 [97338.457152] __kmalloc+0x203/0x250 [97338.457977] virtqueue_add+0x493/0xc70 [97338.458851] virtqueue_add_sgs+0x80/0xa0 [97338.459768] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97338.460993] ? scsi_alloc_sgtables+0x84/0x1c0 [97338.461961] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97338.463040] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97338.464432] scsi_queue_rq+0x512/0xb10 [97338.464440] ll_ost_io00_046: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97338.465378] __blk_mq_try_issue_directly+0x163/0x200 [97338.468815] blk_mq_request_issue_directly+0x4e/0xb0 [97338.469945] blk_mq_try_issue_list_directly+0x62/0x100 [97338.471091] blk_mq_sched_insert_requests+0xa4/0xf0 [97338.472194] blk_mq_flush_plug_list+0x135/0x220 [97338.473249] blk_flush_plug_list+0xd7/0x100 [97338.474222] blk_finish_plug+0x25/0x36 [97338.475112] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97338.476402] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97338.477601] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.478989] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.480336] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97338.481505] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97338.482609] ofd_commitrw+0x5b4/0xd20 [ofd] [97338.483611] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.484786] obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.485905] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97338.487061] ? flush_work+0x42/0x1d0 [97338.487933] ? internal_add_timer+0x42/0x70 [97338.488897] ? _cond_resched+0x15/0x30 [97338.489822] ? mutex_lock+0xe/0x30 [97338.490663] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97338.491951] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97338.493357] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97338.494744] ? finish_wait+0x80/0x80 [97338.495609] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97338.496717] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97338.497956] kthread+0x134/0x150 [97338.498765] ? set_kthread_struct+0x50/0x50 [97338.499721] ret_from_fork+0x1f/0x40 [97338.500579] CPU: 0 PID: 527360 Comm: ll_ost_io00_046 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97338.503798] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97338.505792] Call Trace: [97338.506645] dump_stack+0x41/0x60 [97338.507626] warn_alloc.cold.127+0x7b/0x108 [97338.508584] __alloc_pages_slowpath+0xcb2/0xcd0 [97338.509499] __alloc_pages_nodemask+0x2e2/0x330 [97338.510406] kmalloc_order+0x28/0x90 [97338.511197] kmalloc_order_trace+0x1d/0xb0 [97338.512064] __kmalloc+0x203/0x250 [97338.512929] virtqueue_add+0x493/0xc70 [97338.513832] ? finish_wait+0x80/0x80 [97338.514730] virtqueue_add_sgs+0x80/0xa0 [97338.515511] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97338.516492] ? scsi_alloc_sgtables+0x84/0x1c0 [97338.517315] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97338.518235] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97338.519248] scsi_queue_rq+0x512/0xb10 [97338.520004] __blk_mq_try_issue_directly+0x163/0x200 [97338.520906] blk_mq_request_issue_directly+0x4e/0xb0 [97338.521825] blk_mq_try_issue_list_directly+0x62/0x100 [97338.522873] blk_mq_sched_insert_requests+0xa4/0xf0 [97338.524028] blk_mq_flush_plug_list+0x135/0x220 [97338.524886] blk_flush_plug_list+0xd7/0x100 [97338.525686] blk_finish_plug+0x25/0x36 [97338.526451] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97338.527863] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97338.529248] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.530830] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.532365] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97338.533700] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97338.534933] ofd_commitrw+0x5b4/0xd20 [ofd] [97338.536042] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.537360] obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.538669] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97338.540008] ? flush_work+0x42/0x1d0 [97338.540990] ? common_interrupt+0xa/0xf [97338.542009] ? internal_add_timer+0x42/0x70 [97338.543094] ? _cond_resched+0x15/0x30 [97338.544081] ? mutex_lock+0xe/0x30 [97338.544755] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97338.545758] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97338.546825] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97338.547886] ? finish_wait+0x80/0x80 [97338.548574] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97338.549448] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97338.550416] kthread+0x134/0x150 [97338.551048] ? set_kthread_struct+0x50/0x50 [97338.551819] ret_from_fork+0x1f/0x40 [97338.552527] CPU: 8 PID: 527244 Comm: ll_ost_io02_039 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97338.555001] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97338.557347] Call Trace: [97338.558121] dump_stack+0x41/0x60 [97338.559052] warn_alloc.cold.127+0x7b/0x108 [97338.560118] __alloc_pages_slowpath+0xcb2/0xcd0 [97338.561337] ? __alloc_pages_nodemask+0x166/0x330 [97338.562601] __alloc_pages_nodemask+0x2e2/0x330 [97338.563855] kmalloc_order+0x28/0x90 [97338.564911] kmalloc_order_trace+0x1d/0xb0 [97338.566038] __kmalloc+0x203/0x250 [97338.567010] virtqueue_add+0x493/0xc70 [97338.568030] ? finish_wait+0x80/0x80 [97338.569049] virtqueue_add_sgs+0x80/0xa0 [97338.570117] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97338.571526] ? scsi_alloc_sgtables+0x84/0x1c0 [97338.572712] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97338.574049] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97338.575546] scsi_queue_rq+0x512/0xb10 [97338.576614] __blk_mq_try_issue_directly+0x163/0x200 [97338.577786] blk_mq_request_issue_directly+0x4e/0xb0 [97338.578958] blk_mq_try_issue_list_directly+0x62/0x100 [97338.580208] blk_mq_sched_insert_requests+0xa4/0xf0 [97338.581449] blk_mq_flush_plug_list+0x135/0x220 [97338.582448] blk_flush_plug_list+0xd7/0x100 [97338.583583] blk_finish_plug+0x25/0x36 [97338.584645] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97338.586240] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97338.587737] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.589457] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97338.591115] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97338.592568] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97338.593919] ofd_commitrw+0x5b4/0xd20 [ofd] [97338.595152] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.596578] obd_commitrw+0x1b6/0x370 [ptlrpc] [97338.597619] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97338.598585] ? flush_work+0x42/0x1d0 [97338.599305] ? internal_add_timer+0x42/0x70 [97338.600263] ? _cond_resched+0x15/0x30 [97338.601148] ? mutex_lock+0xe/0x30 [97338.602066] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97338.603488] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97338.605134] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97338.606775] ? lu_context_exit+0x41/0x70 [obdclass] [97338.608158] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97338.609497] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97338.610966] kthread+0x134/0x150 [97338.611929] ? set_kthread_struct+0x50/0x50 [97338.613006] ret_from_fork+0x1f/0x40 [97343.169118] warn_alloc: 1522 callbacks suppressed [97343.169122] ll_ost_io00_020: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97343.173454] CPU: 1 PID: 527192 Comm: ll_ost_io00_020 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97343.176295] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97343.178381] Call Trace: [97343.179194] dump_stack+0x41/0x60 [97343.180190] warn_alloc.cold.127+0x7b/0x108 [97343.181305] __alloc_pages_slowpath+0xcb2/0xcd0 [97343.182465] ? __alloc_pages_nodemask+0x166/0x330 [97343.183648] __alloc_pages_nodemask+0x2e2/0x330 [97343.184697] kmalloc_order+0x28/0x90 [97343.185600] kmalloc_order_trace+0x1d/0xb0 [97343.186663] __kmalloc+0x203/0x250 [97343.187657] virtqueue_add+0x493/0xc70 [97343.188627] ? finish_wait+0x80/0x80 [97343.189615] virtqueue_add_sgs+0x80/0xa0 [97343.190704] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97343.192089] ? scsi_alloc_sgtables+0x84/0x1c0 [97343.193222] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97343.194530] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97343.195956] scsi_queue_rq+0x512/0xb10 [97343.196988] blk_mq_dispatch_rq_list+0x154/0x880 [97343.198219] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [97343.199398] __blk_mq_sched_dispatch_requests+0x110/0x140 [97343.200743] blk_mq_sched_dispatch_requests+0x30/0x60 [97343.202011] __blk_mq_run_hw_queue+0x2b/0x70 [97343.203179] __blk_mq_delay_run_hw_queue+0x141/0x160 [97343.204457] blk_mq_sched_insert_requests+0x6c/0xf0 [97343.205678] blk_mq_flush_plug_list+0x135/0x220 [97343.206865] blk_flush_plug_list+0xd7/0x100 [97343.207989] blk_finish_plug+0x25/0x36 [97343.208971] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97343.210391] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97343.211743] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.213338] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.214849] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97343.216155] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97343.217382] ofd_commitrw+0x5b4/0xd20 [ofd] [97343.218490] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.219786] obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.221035] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97343.222363] ? req_can_reconstruct+0x47/0xf0 [ptlrpc] [97343.223782] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97343.225197] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97343.226715] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97343.228256] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97343.228581] ll_ost_io00_024: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97343.228741] ll_ost_io02_036: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97343.229576] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97343.231980] ,cpuset= [97343.234538] kthread+0x134/0x150 [97343.235974] / [97343.236802] ? set_kthread_struct+0x50/0x50 [97343.237829] ,mems_allowed=0 [97343.238406] ret_from_fork+0x1f/0x40 [97343.240482] warn_alloc_show_mem: 9 callbacks suppressed [97343.240485] Mem-Info: [97343.240483] CPU: 8 PID: 527212 Comm: ll_ost_io02_036 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97343.240488] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97343.240490] Call Trace: [97343.240494] dump_stack+0x41/0x60 [97343.240501] warn_alloc.cold.127+0x7b/0x108 [97343.240507] __alloc_pages_slowpath+0xcb2/0xcd0 [97343.240512] ? __alloc_pages_nodemask+0x166/0x330 [97343.240515] __alloc_pages_nodemask+0x2e2/0x330 [97343.240519] kmalloc_order+0x28/0x90 [97343.240524] kmalloc_order_trace+0x1d/0xb0 [97343.240528] __kmalloc+0x203/0x250 [97343.240533] virtqueue_add+0x493/0xc70 [97343.240540] ? finish_wait+0x80/0x80 [97343.240544] virtqueue_add_sgs+0x80/0xa0 [97343.240550] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97343.240556] ? scsi_alloc_sgtables+0x84/0x1c0 [97343.240561] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97343.240565] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97343.240569] scsi_queue_rq+0x512/0xb10 [97343.240572] __blk_mq_try_issue_directly+0x163/0x200 [97343.240579] blk_mq_request_issue_directly+0x4e/0xb0 [97343.240583] blk_mq_try_issue_list_directly+0x62/0x100 [97343.240587] blk_mq_sched_insert_requests+0xa4/0xf0 [97343.240591] blk_mq_flush_plug_list+0x135/0x220 [97343.240596] blk_flush_plug_list+0xd7/0x100 [97343.240601] blk_finish_plug+0x25/0x36 [97343.240606] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97343.240643] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97343.240674] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.240700] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.240730] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97343.240758] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97343.240779] ofd_commitrw+0x5b4/0xd20 [ofd] [97343.240798] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.240930] obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.241049] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97343.241161] ? flush_work+0x42/0x1d0 [97343.241167] ? internal_add_timer+0x42/0x70 [97343.241171] ? _cond_resched+0x15/0x30 [97343.241173] ? mutex_lock+0xe/0x30 [97343.241177] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97343.241288] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97343.241404] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97343.241512] ? osd_key_exit+0xb/0x100 [osd_ldiskfs] [97343.241535] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97343.242166] active_anon:27952 inactive_anon:200587 isolated_anon:0 active_file:31830829 inactive_file:2935155 isolated_file:32 unevictable:0 dirty:1258 writeback:1 slab_reclaimable:1000935 slab_unreclaimable:605204 mapped:23403 shmem:146 pagetables:5704 bounce:0 free:1410563 free_pcp:468 free_cma:0 [97343.243281] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97343.243999] Node 0 active_anon:111808kB inactive_anon:802348kB active_file:127323316kB inactive_file:11740620kB unevictable:0kB isolated(anon):0kB isolated(file):128kB mapped:93612kB dirty:5032kB writeback:4kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35408kB pagetables:22816kB all_unreclaimable? no [97343.246926] kthread+0x134/0x150 [97343.249018] Node 0 [97343.249948] ? set_kthread_struct+0x50/0x50 [97343.250885] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97343.252105] ret_from_fork+0x1f/0x40 [97343.253310] lowmem_reserve[]: [97343.254606] CPU: 2 PID: 527213 Comm: ll_ost_io00_024 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97343.256197] 0 [97343.257336] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97343.258334] 913 [97343.259395] Call Trace: [97343.260398] 150007 [97343.261188] dump_stack+0x41/0x60 [97343.262130] 150007 [97343.263226] warn_alloc.cold.127+0x7b/0x108 [97343.264232] 150007 [97343.265244] __alloc_pages_slowpath+0xcb2/0xcd0 [97343.267362] ? __alloc_pages_nodemask+0x166/0x330 [97343.268517] Node 0 [97343.269491] __alloc_pages_nodemask+0x2e2/0x330 [97343.270650] DMA32 free:596576kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97343.271619] kmalloc_order+0x28/0x90 [97343.272661] lowmem_reserve[]: [97343.273515] kmalloc_order_trace+0x1d/0xb0 [97343.274449] 0 [97343.275544] __kmalloc+0x203/0x250 [97343.276712] 0 [97343.277880] virtqueue_add+0x493/0xc70 [97343.279232] 149093 [97343.280247] ? finish_wait+0x80/0x80 [97343.281374] 149093 [97343.282289] virtqueue_add_sgs+0x80/0xa0 [97343.283344] 149093 [97343.284297] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97343.285202] ll_ost_io04_036: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97343.286210] ? scsi_alloc_sgtables+0x84/0x1c0 [97343.287201] Node 0 [97343.287989] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97343.288839] Normal free:5022332kB min:49268kB low:201940kB high:354612kB active_anon:111428kB inactive_anon:802344kB active_file:127317084kB inactive_file:11760044kB unevictable:0kB writepending:5856kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1948kB local_pcp:48kB free_cma:0kB [97343.289353] ll_ost_io02_001: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97343.289861] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97343.291103] lowmem_reserve[]: [97343.292215] scsi_queue_rq+0x512/0xb10 [97343.293323] 0 [97343.294364] blk_mq_dispatch_rq_list+0x154/0x880 [97343.301247] 0 [97343.302210] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [97343.308090] 0 [97343.308797] __blk_mq_sched_dispatch_requests+0x110/0x140 [97343.309442] 0 [97343.310314] blk_mq_sched_dispatch_requests+0x30/0x60 [97343.314580] 0 [97343.315658] __blk_mq_run_hw_queue+0x2b/0x70 [97343.319454] __blk_mq_delay_run_hw_queue+0x141/0x160 [97343.320036] Node 0 [97343.322306] blk_mq_sched_insert_requests+0x6c/0xf0 [97343.322931] DMA: [97343.323812] blk_mq_flush_plug_list+0x135/0x220 [97343.324469] 0*4kB [97343.325499] blk_flush_plug_list+0xd7/0x100 [97343.326155] 0*8kB [97343.327394] blk_finish_plug+0x25/0x36 [97343.328037] 0*16kB [97343.329329] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97343.329889] 0*32kB [97343.331245] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97343.331913] 0*64kB [97343.333231] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.337835] 0*128kB [97343.338904] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.339699] 0*256kB [97343.340888] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97343.341485] 0*512kB [97343.342550] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97343.343116] 1*1024kB [97343.344240] ofd_commitrw+0x5b4/0xd20 [ofd] [97343.344868] (U) [97343.345953] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.346569] 1*2048kB [97343.347746] obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.348362] (M) [97343.349821] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97343.352400] 2*4096kB [97343.353136] ? flush_work+0x42/0x1d0 [97343.354192] (M) [97343.354907] ? internal_add_timer+0x42/0x70 [97343.356075] = 11264kB [97343.362076] ? _cond_resched+0x15/0x30 [97343.364562] Node 0 [97343.365887] ? mutex_lock+0xe/0x30 [97343.366688] DMA32: [97343.367862] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97343.368431] 18*4kB [97343.369532] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97343.370101] (UMH) [97343.371030] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97343.371581] 21*8kB [97343.372636] ? lu_context_exit+0x41/0x70 [obdclass] [97343.373196] (UMH) [97343.374171] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97343.374725] 29*16kB [97343.375605] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97343.376136] (UMEH) [97343.377122] kthread+0x134/0x150 [97343.377759] 51*32kB [97343.378721] ? set_kthread_struct+0x50/0x50 [97343.379321] (UMH) [97343.380241] ret_from_fork+0x1f/0x40 [97343.380852] 49*64kB [97343.381693] CPU: 16 PID: 527548 Comm: ll_ost_io04_036 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97343.382361] (UMH) [97343.383472] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97343.384218] 47*128kB [97343.385742] Call Trace: [97343.386402] (UM) [97343.387793] dump_stack+0x41/0x60 [97343.388470] 44*256kB [97343.390101] warn_alloc.cold.127+0x7b/0x108 [97343.390744] (UM) [97343.392336] __alloc_pages_slowpath+0xcb2/0xcd0 [97343.393131] 45*512kB [97343.394488] ? __alloc_pages_nodemask+0x166/0x330 [97343.395151] (UMEH) [97343.396517] __alloc_pages_nodemask+0x2e2/0x330 [97343.397201] 38*1024kB [97343.398412] kmalloc_order+0x28/0x90 [97343.399019] (UMH) [97343.400354] kmalloc_order_trace+0x1d/0xb0 [97343.401017] 42*2048kB [97343.402310] __kmalloc+0x203/0x250 [97343.402928] (UM) [97343.404234] virtqueue_add+0x493/0xc70 [97343.404934] 104*4096kB [97343.405981] ? finish_wait+0x80/0x80 [97343.406600] (UM) [97343.407775] virtqueue_add_sgs+0x80/0xa0 [97343.408445] = 596704kB [97343.409562] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97343.410169] Node 0 [97343.411176] ? scsi_alloc_sgtables+0x84/0x1c0 [97343.411779] Normal: [97343.413123] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97343.413730] 679407*4kB [97343.415196] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97343.415783] (UME) [97343.417107] scsi_queue_rq+0x512/0xb10 [97343.417713] 263763*8kB [97343.419010] blk_mq_dispatch_rq_list+0x154/0x880 [97343.419607] (UME) [97343.420812] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [97343.421442] 5914*16kB [97343.422764] __blk_mq_sched_dispatch_requests+0x110/0x140 [97343.423368] (UMEH) [97343.424315] blk_mq_sched_dispatch_requests+0x30/0x60 [97343.424931] 13*32kB [97343.426098] __blk_mq_run_hw_queue+0x2b/0x70 [97343.426682] (UEH) [97343.427773] __blk_mq_delay_run_hw_queue+0x141/0x160 [97343.428335] 1*64kB (U) [97343.431266] blk_mq_sched_insert_requests+0x6c/0xf0 [97343.431851] 0*128kB [97343.434103] blk_mq_flush_plug_list+0x135/0x220 [97343.434761] 0*256kB [97343.435612] blk_flush_plug_list+0xd7/0x100 [97343.436191] 0*512kB [97343.437190] blk_finish_plug+0x25/0x36 [97343.437830] 0*1024kB [97343.439020] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97343.439591] 0*2048kB [97343.440862] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97343.441502] 0*4096kB [97343.442825] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.443432] = 4922836kB [97343.444703] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.445359] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97343.446428] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97343.447001] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97343.448174] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97343.448801] 34800570 total pagecache pages [97343.449820] ofd_commitrw+0x5b4/0xd20 [ofd] [97343.450388] 1620 pages in swap cache [97343.451493] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.452115] Swap cache stats: add 1608490, delete 1606893, find 83808/101122 [97343.453179] obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.453737] Free swap = 5331516kB [97343.454905] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97343.455534] Total swap = 11075580kB [97343.456991] ? flush_work+0x42/0x1d0 [97343.457599] 39321462 pages RAM [97343.458864] ? internal_add_timer+0x42/0x70 [97343.459498] 0 pages HighMem/MovableOnly [97343.460870] ? _cond_resched+0x15/0x30 [97343.461551] 905848 pages reserved [97343.463064] ? mutex_lock+0xe/0x30 [97343.463687] 0 pages hwpoisoned [97343.464814] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97343.515828] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97343.517039] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97343.518584] ? finish_wait+0x80/0x80 [97343.519544] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97343.520811] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97343.522200] kthread+0x134/0x150 [97343.523137] ? set_kthread_struct+0x50/0x50 [97343.524239] ret_from_fork+0x1f/0x40 [97343.525207] CPU: 8 PID: 188104 Comm: ll_ost_io02_001 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97343.527704] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97343.529757] Call Trace: [97343.530532] dump_stack+0x41/0x60 [97343.531478] warn_alloc.cold.127+0x7b/0x108 [97343.532640] __alloc_pages_slowpath+0xcb2/0xcd0 [97343.533871] ? __alloc_pages_nodemask+0x166/0x330 [97343.535113] __alloc_pages_nodemask+0x2e2/0x330 [97343.536152] kmalloc_order+0x28/0x90 [97343.537040] kmalloc_order_trace+0x1d/0xb0 [97343.538126] __kmalloc+0x203/0x250 [97343.539079] virtqueue_add+0x493/0xc70 [97343.540106] virtqueue_add_sgs+0x80/0xa0 [97343.541171] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97343.542535] ? scsi_alloc_sgtables+0x84/0x1c0 [97343.543689] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97343.544992] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97343.546360] scsi_queue_rq+0x512/0xb10 [97343.547415] __blk_mq_try_issue_directly+0x163/0x200 [97343.548703] blk_mq_request_issue_directly+0x4e/0xb0 [97343.549979] blk_mq_try_issue_list_directly+0x62/0x100 [97343.551288] blk_mq_sched_insert_requests+0xa4/0xf0 [97343.552568] blk_mq_flush_plug_list+0x135/0x220 [97343.553755] blk_flush_plug_list+0xd7/0x100 [97343.554891] blk_finish_plug+0x25/0x36 [97343.555935] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97343.557430] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97343.558803] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.560430] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97343.562069] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97343.563443] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97343.564719] ofd_commitrw+0x5b4/0xd20 [ofd] [97343.565904] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.567251] obd_commitrw+0x1b6/0x370 [ptlrpc] [97343.568595] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97343.569974] ? flush_work+0x42/0x1d0 [97343.570997] ? internal_add_timer+0x42/0x70 [97343.572154] ? _cond_resched+0x15/0x30 [97343.573183] ? mutex_lock+0xe/0x30 [97343.574148] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97343.575576] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97343.577112] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97343.578688] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97343.579963] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97343.581081] kthread+0x134/0x150 [97343.581876] ? set_kthread_struct+0x50/0x50 [97343.582972] ret_from_fork+0x1f/0x40 [97364.024657] ll_ost_io00_030: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97364.026286] ll_ost_io01_044: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97364.027020] ll_ost_io03_022: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97364.027032] CPU: 13 PID: 527061 Comm: ll_ost_io03_022 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97364.027036] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97364.027038] Call Trace: [97364.027043] dump_stack+0x41/0x60 [97364.027051] warn_alloc.cold.127+0x7b/0x108 [97364.027058] __alloc_pages_slowpath+0xcb2/0xcd0 [97364.027062] ? __alloc_pages_nodemask+0x166/0x330 [97364.027065] __alloc_pages_nodemask+0x2e2/0x330 [97364.027069] kmalloc_order+0x28/0x90 [97364.027074] kmalloc_order_trace+0x1d/0xb0 [97364.027078] __kmalloc+0x203/0x250 [97364.027084] virtqueue_add+0x493/0xc70 [97364.027091] virtqueue_add_sgs+0x80/0xa0 [97364.027096] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97364.027102] ? scsi_alloc_sgtables+0x84/0x1c0 [97364.027108] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97364.027112] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97364.027116] scsi_queue_rq+0x512/0xb10 [97364.027119] __blk_mq_try_issue_directly+0x163/0x200 [97364.027126] blk_mq_request_issue_directly+0x4e/0xb0 [97364.027130] blk_mq_try_issue_list_directly+0x62/0x100 [97364.027134] blk_mq_sched_insert_requests+0xa4/0xf0 [97364.027138] blk_mq_flush_plug_list+0x135/0x220 [97364.027142] blk_flush_plug_list+0xd7/0x100 [97364.027148] blk_finish_plug+0x25/0x36 [97364.027153] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97364.027191] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97364.027224] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.027251] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.027279] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97364.027308] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97364.027332] ofd_commitrw+0x5b4/0xd20 [ofd] [97364.027351] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.027476] obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.027601] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97364.028357] ll_ost_io03_031: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97364.030267] ,cpuset= [97364.032899] ? flush_work+0x42/0x1d0 [97364.035915] / [97364.038176] ? internal_add_timer+0x42/0x70 [97364.039070] ,mems_allowed=0 [97364.040099] ? _cond_resched+0x15/0x30 [97364.042581] ? mutex_lock+0xe/0x30 [97364.084943] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97364.086356] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97364.087708] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97364.089156] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97364.090431] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97364.091586] kthread+0x134/0x150 [97364.092496] ? set_kthread_struct+0x50/0x50 [97364.093597] ret_from_fork+0x1f/0x40 [97364.094601] warn_alloc_show_mem: 4 callbacks suppressed [97364.094605] Mem-Info: [97364.094602] CPU: 4 PID: 527148 Comm: ll_ost_io01_044 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97364.095934] active_anon:27867 inactive_anon:200016 isolated_anon:0 active_file:30969710 inactive_file:3531175 isolated_file:0 unevictable:0 dirty:1666 writeback:1 slab_reclaimable:1000907 slab_unreclaimable:604322 mapped:23496 shmem:146 pagetables:5704 bounce:0 free:1676491 free_pcp:402 free_cma:0 [97364.096740] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97364.099385] Node 0 active_anon:111468kB inactive_anon:800064kB active_file:123878840kB inactive_file:14124700kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:93984kB dirty:6664kB writeback:4kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35424kB pagetables:22816kB all_unreclaimable? no [97364.105824] Call Trace: [97364.107882] Node 0 [97364.113521] dump_stack+0x41/0x60 [97364.114277] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97364.114875] warn_alloc.cold.127+0x7b/0x108 [97364.115566] lowmem_reserve[]: [97364.119174] __alloc_pages_slowpath+0xcb2/0xcd0 [97364.120368] 0 [97364.121040] ? __alloc_pages_nodemask+0x166/0x330 [97364.122306] 913 [97364.122808] __alloc_pages_nodemask+0x2e2/0x330 [97364.124128] 150007 [97364.124664] kmalloc_order+0x28/0x90 [97364.125961] 150007 [97364.126534] kmalloc_order_trace+0x1d/0xb0 [97364.127604] 150007 [97364.128203] __kmalloc+0x203/0x250 [97364.129809] virtqueue_add+0x493/0xc70 [97364.130688] Node 0 [97364.131156] virtqueue_add_sgs+0x80/0xa0 [97364.132231] DMA32 free:596512kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97364.132770] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97364.133748] lowmem_reserve[]: [97364.137613] ? scsi_alloc_sgtables+0x84/0x1c0 [97364.138913] 0 [97364.139597] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97364.140831] 0 [97364.141373] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97364.142730] 149093 [97364.143245] scsi_queue_rq+0x512/0xb10 [97364.144751] 149093 [97364.145318] __blk_mq_try_issue_directly+0x163/0x200 [97364.146430] 149093 [97364.147014] blk_mq_request_issue_directly+0x4e/0xb0 [97364.148911] blk_mq_try_issue_list_directly+0x62/0x100 [97364.150151] Node 0 [97364.150639] blk_mq_sched_insert_requests+0xa4/0xf0 [97364.151965] Normal free:6081952kB min:49268kB low:201940kB high:354612kB active_anon:111088kB inactive_anon:800060kB active_file:123898576kB inactive_file:14121296kB unevictable:0kB writepending:6668kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1992kB local_pcp:128kB free_cma:0kB [97364.152521] blk_mq_flush_plug_list+0x135/0x220 [97364.153861] lowmem_reserve[]: [97364.158117] blk_flush_plug_list+0xd7/0x100 [97364.159286] 0 [97364.159955] blk_finish_plug+0x25/0x36 [97364.160949] 0 [97364.161469] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97364.162316] 0 [97364.162806] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97364.164113] 0 [97364.164588] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.165782] 0 [97364.166253] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.168079] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97364.169335] Node 0 [97364.169784] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97364.171134] DMA: [97364.171652] ofd_commitrw+0x5b4/0xd20 [ofd] [97364.172701] 0*4kB [97364.173196] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.174100] 0*8kB [97364.174609] obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.175533] 0*16kB [97364.176069] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97364.177010] 0*32kB [97364.177557] ? flush_work+0x42/0x1d0 [97364.178601] 0*64kB [97364.179153] ? internal_add_timer+0x42/0x70 [97364.180021] 0*128kB [97364.180568] ? _cond_resched+0x15/0x30 [97364.181581] 0*256kB [97364.182132] ? mutex_lock+0xe/0x30 [97364.183030] 0*512kB [97364.183577] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97364.184539] 1*1024kB [97364.185076] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97364.186098] (U) [97364.186660] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97364.187778] 1*2048kB [97364.188280] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97364.189478] (M) [97364.190028] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97364.190988] 2*4096kB [97364.191487] kthread+0x134/0x150 [97364.192541] (M) [97364.193099] ? set_kthread_struct+0x50/0x50 [97364.193886] = 11264kB [97364.194366] ret_from_fork+0x1f/0x40 [97364.195521] Node 0 [97364.196052] CPU: 2 PID: 527262 Comm: ll_ost_io00_030 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97364.198228] DMA32: [97364.198961] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97364.201856] 18*4kB [97364.202595] Call Trace: [97364.204524] (UM) [97364.205279] dump_stack+0x41/0x60 [97364.205896] 21*8kB [97364.206626] warn_alloc.cold.127+0x7b/0x108 [97364.207377] (UM) [97364.208164] __alloc_pages_slowpath+0xcb2/0xcd0 [97364.209060] 29*16kB [97364.209604] ? __alloc_pages_nodemask+0x166/0x330 [97364.210811] (UME) [97364.211383] __alloc_pages_nodemask+0x2e2/0x330 [97364.212544] 41*32kB [97364.213240] kmalloc_order+0x28/0x90 [97364.214377] (UMH) [97364.215097] kmalloc_order_trace+0x1d/0xb0 [97364.216084] 45*64kB [97364.216797] __kmalloc+0x203/0x250 [97364.217916] (UM) [97364.218678] virtqueue_add+0x493/0xc70 [97364.219667] 46*128kB [97364.220366] ? finish_wait+0x80/0x80 [97364.221327] (UM) [97364.222077] virtqueue_add_sgs+0x80/0xa0 [97364.223076] 44*256kB [97364.223793] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97364.224749] (UM) [97364.225531] ? scsi_alloc_sgtables+0x84/0x1c0 [97364.226883] 46*512kB [97364.227588] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97364.228709] (UMEH) [97364.229490] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97364.230534] 38*1024kB [97364.231272] scsi_queue_rq+0x512/0xb10 [97364.232354] (UM) [97364.233147] __blk_mq_try_issue_directly+0x163/0x200 [97364.233931] 42*2048kB [97364.234637] blk_mq_request_issue_directly+0x4e/0xb0 [97364.235601] (UM) [97364.236443] blk_mq_try_issue_list_directly+0x62/0x100 [97364.237331] 104*4096kB [97364.238028] blk_mq_sched_insert_requests+0xa4/0xf0 [97364.239237] (UM) [97364.240048] blk_mq_flush_plug_list+0x135/0x220 [97364.241296] = 596512kB [97364.241935] blk_flush_plug_list+0xd7/0x100 [97364.243052] Node 0 [97364.243651] blk_finish_plug+0x25/0x36 [97364.244477] Normal: [97364.245009] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97364.245806] 1076991*4kB [97364.246460] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97364.247498] (UMEH) [97364.248091] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.249163] 197956*8kB [97364.249665] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.250860] (UMEH) [97364.251442] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97364.252574] 6812*16kB [97364.253086] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97364.254153] (UMEH) [97364.254709] ofd_commitrw+0x5b4/0xd20 [ofd] [97364.255660] 17*32kB [97364.256172] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.257164] (UEH) [97364.257682] obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.258568] 2*64kB [97364.259153] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97364.260036] (H) [97364.260720] ? flush_work+0x42/0x1d0 [97364.261688] 0*128kB [97364.262212] ? internal_add_timer+0x42/0x70 [97364.262983] 0*256kB [97364.263686] ? _cond_resched+0x15/0x30 [97364.264595] 0*512kB [97364.265249] ? mutex_lock+0xe/0x30 [97364.265991] 0*1024kB [97364.266723] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97364.267382] 0*2048kB [97364.268110] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97364.268998] 0*4096kB [97364.269745] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97364.270784] = 6001276kB [97364.271532] ? tgt_key_exit+0xc/0x20 [ptlrpc] [97364.272568] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97364.273362] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97364.274190] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97364.276167] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97364.277155] 34528035 total pagecache pages [97364.279093] kthread+0x134/0x150 [97364.280210] 1743 pages in swap cache [97364.281351] ? set_kthread_struct+0x50/0x50 [97364.282289] Swap cache stats: add 1609787, delete 1608067, find 83909/101242 [97364.283298] ret_from_fork+0x1f/0x40 [97364.284285] Free swap = 5327404kB [97364.285779] CPU: 15 PID: 527231 Comm: ll_ost_io03_031 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97364.286808] Total swap = 11075580kB [97364.287863] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97364.290807] 39321462 pages RAM [97364.291887] Call Trace: [97364.294009] 0 pages HighMem/MovableOnly [97364.294999] dump_stack+0x41/0x60 [97364.295863] 905848 pages reserved [97364.297007] warn_alloc.cold.127+0x7b/0x108 [97364.298013] 0 pages hwpoisoned [97364.299041] __alloc_pages_slowpath+0xcb2/0xcd0 [97364.302342] ? __alloc_pages_nodemask+0x166/0x330 [97364.303562] __alloc_pages_nodemask+0x2e2/0x330 [97364.304727] kmalloc_order+0x28/0x90 [97364.305705] kmalloc_order_trace+0x1d/0xb0 [97364.306768] __kmalloc+0x203/0x250 [97364.307704] virtqueue_add+0x493/0xc70 [97364.308746] ? finish_wait+0x80/0x80 [97364.309729] virtqueue_add_sgs+0x80/0xa0 [97364.310786] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97364.312126] ? scsi_alloc_sgtables+0x84/0x1c0 [97364.313252] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97364.314562] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97364.316021] scsi_queue_rq+0x512/0xb10 [97364.317091] __blk_mq_try_issue_directly+0x163/0x200 [97364.318404] blk_mq_request_issue_directly+0x4e/0xb0 [97364.319717] blk_mq_try_issue_list_directly+0x62/0x100 [97364.321064] blk_mq_sched_insert_requests+0xa4/0xf0 [97364.322343] blk_mq_flush_plug_list+0x135/0x220 [97364.323578] blk_flush_plug_list+0xd7/0x100 [97364.324721] blk_finish_plug+0x25/0x36 [97364.325790] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97364.327309] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97364.328740] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.330354] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97364.331909] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97364.333337] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97364.334597] ofd_commitrw+0x5b4/0xd20 [ofd] [97364.335763] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.337152] obd_commitrw+0x1b6/0x370 [ptlrpc] [97364.338514] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97364.339934] ? flush_work+0x42/0x1d0 [97364.341036] ? internal_add_timer+0x42/0x70 [97364.342243] ? _cond_resched+0x15/0x30 [97364.343343] ? mutex_lock+0xe/0x30 [97364.344452] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97364.345960] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97364.347607] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97364.349315] ? finish_wait+0x80/0x80 [97364.350381] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97364.351717] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97364.353153] kthread+0x134/0x150 [97364.354074] ? set_kthread_struct+0x50/0x50 [97364.355180] ret_from_fork+0x1f/0x40 [97367.827535] ll_ost_io03_070: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97367.828828] ll_ost_io02_053: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [97367.830437] CPU: 14 PID: 527455 Comm: ll_ost_io03_070 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97367.830443] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97367.833132] ,mems_allowed=0 [97367.833664] ll_ost_io00_002: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97367.834778] ll_ost_io02_008: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97367.836084] Call Trace: [97367.837065] ll_ost_io04_049: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97367.839311] dump_stack+0x41/0x60 [97367.841118] ll_ost_io00_058: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97367.853079] warn_alloc.cold.127+0x7b/0x108 [97367.854301] __alloc_pages_slowpath+0xcb2/0xcd0 [97367.855583] ? __alloc_pages_nodemask+0x166/0x330 [97367.856838] __alloc_pages_nodemask+0x2e2/0x330 [97367.858026] kmalloc_order+0x28/0x90 [97367.859040] kmalloc_order_trace+0x1d/0xb0 [97367.860139] __kmalloc+0x203/0x250 [97367.861134] virtqueue_add+0x493/0xc70 [97367.862150] ? finish_wait+0x80/0x80 [97367.863151] virtqueue_add_sgs+0x80/0xa0 [97367.864206] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97367.865539] ? scsi_alloc_sgtables+0x84/0x1c0 [97367.866727] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97367.868047] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97367.869449] scsi_queue_rq+0x512/0xb10 [97367.870501] __blk_mq_try_issue_directly+0x163/0x200 [97367.871782] blk_mq_request_issue_directly+0x4e/0xb0 [97367.873041] blk_mq_try_issue_list_directly+0x62/0x100 [97367.874394] blk_mq_sched_insert_requests+0xa4/0xf0 [97367.875645] blk_mq_flush_plug_list+0x135/0x220 [97367.876847] blk_flush_plug_list+0xd7/0x100 [97367.877992] blk_finish_plug+0x25/0x36 [97367.879110] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97367.880551] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97367.881892] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97367.883430] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97367.884946] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97367.886276] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97367.887507] ofd_commitrw+0x5b4/0xd20 [ofd] [97367.888639] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97367.889960] obd_commitrw+0x1b6/0x370 [ptlrpc] [97367.891230] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97367.892559] ? flush_work+0x42/0x1d0 [97367.893593] ? internal_add_timer+0x42/0x70 [97367.894713] ? _cond_resched+0x15/0x30 [97367.895716] ? mutex_lock+0xe/0x30 [97367.896659] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97367.898058] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97367.899577] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97367.901099] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97367.902409] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97367.903788] kthread+0x134/0x150 [97367.904688] ? set_kthread_struct+0x50/0x50 [97367.905778] ret_from_fork+0x1f/0x40 [97367.906739] warn_alloc_show_mem: 3 callbacks suppressed [97367.906740] CPU: 1 PID: 527435 Comm: ll_ost_io00_058 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97367.906742] Mem-Info: [97367.906744] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97367.908041] active_anon:27804 inactive_anon:199813 isolated_anon:0 active_file:31698960 inactive_file:3420696 isolated_file:39 unevictable:0 dirty:743 writeback:0 slab_reclaimable:1000907 slab_unreclaimable:604430 mapped:23605 shmem:146 pagetables:5744 bounce:0 free:1057831 free_pcp:350 free_cma:0 [97367.910547] Call Trace: [97367.911358] Node 0 active_anon:111216kB inactive_anon:799252kB active_file:126795840kB inactive_file:13682784kB unevictable:0kB isolated(anon):0kB isolated(file):156kB mapped:94420kB dirty:2972kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35424kB pagetables:22976kB all_unreclaimable? no [97367.913227] dump_stack+0x41/0x60 [97367.921138] Node 0 [97367.921828] warn_alloc.cold.127+0x7b/0x108 [97367.928568] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97367.929400] __alloc_pages_slowpath+0xcb2/0xcd0 [97367.930147] lowmem_reserve[]: [97367.931118] ? __alloc_pages_nodemask+0x166/0x330 [97367.935937] 0 [97367.936977] __alloc_pages_nodemask+0x2e2/0x330 [97367.937909] 913 [97367.938985] kmalloc_order+0x28/0x90 [97367.939663] 150007 [97367.940703] kmalloc_order_trace+0x1d/0xb0 [97367.941445] 150007 [97367.942335] __kmalloc+0x203/0x250 [97367.943123] 150007 [97367.944275] virtqueue_add+0x493/0xc70 [97367.945964] ? finish_wait+0x80/0x80 [97367.946746] Node 0 [97367.947652] virtqueue_add_sgs+0x80/0xa0 [97367.948372] DMA32 free:596544kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97367.949292] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97367.950077] lowmem_reserve[]: [97367.951053] ? scsi_alloc_sgtables+0x84/0x1c0 [97367.956292] 0 [97367.957500] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97367.958464] 0 [97367.959519] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97367.960223] 149093 [97367.961501] scsi_queue_rq+0x512/0xb10 [97367.962191] 149093 [97367.963445] __blk_mq_try_issue_directly+0x163/0x200 [97367.964225] 149093 [97367.965164] blk_mq_request_issue_directly+0x4e/0xb0 [97367.967096] blk_mq_try_issue_list_directly+0x62/0x100 [97367.967881] Node 0 [97367.969019] blk_mq_sched_insert_requests+0xa4/0xf0 [97367.969689] Normal free:3590884kB min:49268kB low:201940kB high:354612kB active_anon:110836kB inactive_anon:799196kB active_file:126785240kB inactive_file:13726200kB unevictable:0kB writepending:2996kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1348kB local_pcp:140kB free_cma:0kB [97367.970858] blk_mq_flush_plug_list+0x135/0x220 [97367.971628] lowmem_reserve[]: [97367.972756] blk_flush_plug_list+0xd7/0x100 [97367.978581] 0 [97367.979656] blk_finish_plug+0x25/0x36 [97367.980593] 0 [97367.981584] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97367.982255] 0 [97367.983171] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97367.983822] 0 [97367.985082] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97367.985794] 0 [97367.986960] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97367.988945] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97367.989595] Node 0 [97367.990924] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97367.991546] DMA: [97367.992697] ofd_commitrw+0x5b4/0xd20 [ofd] [97367.993456] 0*4kB [97367.994554] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97367.995253] 0*8kB [97367.996221] obd_commitrw+0x1b6/0x370 [ptlrpc] [97367.996915] 0*16kB [97367.997980] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97367.998680] 0*32kB [97367.999703] ? flush_work+0x42/0x1d0 [97368.000467] 0*64kB [97368.001576] ? internal_add_timer+0x42/0x70 [97368.002362] 0*128kB [97368.003273] ? _cond_resched+0x15/0x30 [97368.004050] 0*256kB [97368.005033] ? mutex_lock+0xe/0x30 [97368.005812] 0*512kB [97368.006733] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97368.007462] 1*1024kB [97368.008322] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97368.009047] (U) [97368.010175] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97368.010915] 1*2048kB [97368.012161] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97368.012825] (M) [97368.014073] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97368.014820] 2*4096kB [97368.015847] kthread+0x134/0x150 [97368.016547] (M) [97368.017699] ? set_kthread_struct+0x50/0x50 [97368.018490] = 11264kB [97368.019320] ret_from_fork+0x1f/0x40 [97368.020009] Node 0 [97368.020987] CPU: 11 PID: 527031 Comm: ll_ost_io02_008 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97368.021693] DMA32: [97368.022446] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97368.023179] 18*4kB [97368.025380] Call Trace: [97368.026155] (UM) [97368.027700] dump_stack+0x41/0x60 [97368.028448] 21*8kB [97368.029021] warn_alloc.cold.127+0x7b/0x108 [97368.029730] (UM) [97368.030408] __alloc_pages_slowpath+0xcb2/0xcd0 [97368.031146] 29*16kB [97368.031956] ? __alloc_pages_nodemask+0x166/0x330 [97368.032645] (UME) [97368.033495] __alloc_pages_nodemask+0x2e2/0x330 [97368.034251] 43*32kB [97368.035184] kmalloc_order+0x28/0x90 [97368.035911] (UMH) [97368.036793] kmalloc_order_trace+0x1d/0xb0 [97368.037450] 45*64kB [97368.038167] __kmalloc+0x203/0x250 [97368.038721] (UM) [97368.039769] ? __wake_up_common+0x7a/0x190 [97368.040361] 46*128kB [97368.041107] virtqueue_add+0x493/0xc70 [97368.041676] (UM) [97368.042504] ? finish_wait+0x80/0x80 [97368.043258] 44*256kB [97368.043990] virtqueue_add_sgs+0x80/0xa0 [97368.044709] (UM) [97368.045658] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97368.046451] 46*512kB [97368.047250] ? scsi_alloc_sgtables+0x84/0x1c0 [97368.047976] (UMEH) [97368.048956] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97368.049743] 38*1024kB [97368.050591] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97368.051329] (UM) [97368.052260] scsi_queue_rq+0x512/0xb10 [97368.053049] 42*2048kB [97368.054065] __blk_mq_try_issue_directly+0x163/0x200 [97368.054775] (UM) [97368.055549] blk_mq_request_issue_directly+0x4e/0xb0 [97368.056319] 104*4096kB [97368.057226] blk_mq_try_issue_list_directly+0x62/0x100 [97368.057936] (UM) [97368.058860] blk_mq_sched_insert_requests+0xa4/0xf0 [97368.059679] = 596576kB [97368.060696] blk_mq_flush_plug_list+0x135/0x220 [97368.061388] Node 0 [97368.062317] blk_flush_plug_list+0xd7/0x100 [97368.063124] Normal: [97368.063979] blk_finish_plug+0x25/0x36 [97368.064725] 425937*4kB [97368.065530] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97368.066270] (UMEH) [97368.067287] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97368.068027] 214915*8kB [97368.069335] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.070046] (UMEH) [97368.070991] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.071770] 7121*16kB [97368.072882] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97368.073599] (UMEH) [97368.074819] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97368.075545] 2*32kB [97368.076522] ofd_commitrw+0x5b4/0xd20 [ofd] [97368.077245] (H) [97368.078095] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.078781] 0*64kB [97368.079571] obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.080207] 0*128kB [97368.081078] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97368.081761] 0*256kB [97368.082610] ? flush_work+0x42/0x1d0 [97368.083359] 0*512kB [97368.084234] ? internal_add_timer+0x42/0x70 [97368.084986] 0*1024kB [97368.085689] ? _cond_resched+0x15/0x30 [97368.086585] 0*2048kB [97368.087382] ? mutex_lock+0xe/0x30 [97368.088154] 0*4096kB [97368.088929] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97368.089664] = 3537068kB [97368.090375] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97368.091110] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97368.092089] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97368.092853] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97368.093920] ? osd_it_ea_store+0x20/0x20 [osd_ldiskfs] [97368.095909] 35152279 total pagecache pages [97368.097001] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97368.098935] 1609 pages in swap cache [97368.100038] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97368.100985] Swap cache stats: add 1609932, delete 1608338, find 83909/101242 [97368.101861] kthread+0x134/0x150 [97368.102801] Free swap = 5326892kB [97368.103740] ? set_kthread_struct+0x50/0x50 [97368.105197] Total swap = 11075580kB [97368.105873] ret_from_fork+0x1f/0x40 [97368.106624] 39321462 pages RAM [97368.107428] CPU: 9 PID: 527317 Comm: ll_ost_io02_053 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97368.108155] 0 pages HighMem/MovableOnly [97368.109000] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97368.109722] 905848 pages reserved [97368.111744] Call Trace: [97368.112543] 0 pages hwpoisoned [97368.114098] dump_stack+0x41/0x60 [97368.117287] warn_alloc.cold.127+0x7b/0x108 [97368.118299] __alloc_pages_slowpath+0xcb2/0xcd0 [97368.119502] ? __alloc_pages_nodemask+0x166/0x330 [97368.120761] __alloc_pages_nodemask+0x2e2/0x330 [97368.121980] kmalloc_order+0x28/0x90 [97368.122996] kmalloc_order_trace+0x1d/0xb0 [97368.124076] __kmalloc+0x203/0x250 [97368.125047] ? __wake_up_common+0x7a/0x190 [97368.126145] virtqueue_add+0x493/0xc70 [97368.127162] virtqueue_add_sgs+0x80/0xa0 [97368.128183] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97368.129403] ? scsi_alloc_sgtables+0x84/0x1c0 [97368.130213] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97368.131111] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97368.132080] scsi_queue_rq+0x512/0xb10 [97368.132786] __blk_mq_try_issue_directly+0x163/0x200 [97368.133697] blk_mq_request_issue_directly+0x4e/0xb0 [97368.134613] blk_mq_try_issue_list_directly+0x62/0x100 [97368.135510] blk_mq_sched_insert_requests+0xa4/0xf0 [97368.136420] blk_mq_flush_plug_list+0x135/0x220 [97368.137306] blk_flush_plug_list+0xd7/0x100 [97368.138096] blk_finish_plug+0x25/0x36 [97368.138810] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97368.139857] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97368.140842] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.141986] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.143055] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97368.143989] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97368.144910] ofd_commitrw+0x5b4/0xd20 [ofd] [97368.145707] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.146673] obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.147577] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97368.148481] ? flush_work+0x42/0x1d0 [97368.149177] ? internal_add_timer+0x42/0x70 [97368.149951] ? _cond_resched+0x15/0x30 [97368.150652] ? mutex_lock+0xe/0x30 [97368.151304] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97368.152281] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97368.153380] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97368.154549] ? tgt_key_exit+0xc/0x20 [ptlrpc] [97368.155504] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97368.156445] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97368.157695] kthread+0x134/0x150 [97368.158425] ? set_kthread_struct+0x50/0x50 [97368.159192] ret_from_fork+0x1f/0x40 [97368.159885] CPU: 18 PID: 527686 Comm: ll_ost_io04_049 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97368.162710] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97368.164826] Call Trace: [97368.165656] dump_stack+0x41/0x60 [97368.166642] warn_alloc.cold.127+0x7b/0x108 [97368.167758] __alloc_pages_slowpath+0xcb2/0xcd0 [97368.168926] ? __alloc_pages_nodemask+0x166/0x330 [97368.170108] __alloc_pages_nodemask+0x2e2/0x330 [97368.171264] kmalloc_order+0x28/0x90 [97368.172233] kmalloc_order_trace+0x1d/0xb0 [97368.173292] __kmalloc+0x203/0x250 [97368.174214] virtqueue_add+0x493/0xc70 [97368.175204] ? finish_wait+0x80/0x80 [97368.176150] virtqueue_add_sgs+0x80/0xa0 [97368.177189] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97368.178490] ? scsi_alloc_sgtables+0x84/0x1c0 [97368.179582] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97368.180808] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97368.182207] scsi_queue_rq+0x512/0xb10 [97368.183241] __blk_mq_try_issue_directly+0x163/0x200 [97368.184464] blk_mq_request_issue_directly+0x4e/0xb0 [97368.185726] blk_mq_try_issue_list_directly+0x62/0x100 [97368.187012] blk_mq_sched_insert_requests+0xa4/0xf0 [97368.188240] blk_mq_flush_plug_list+0x135/0x220 [97368.189398] blk_flush_plug_list+0xd7/0x100 [97368.190476] blk_finish_plug+0x25/0x36 [97368.191486] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97368.192917] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97368.194231] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.195763] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.197283] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97368.198610] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97368.199865] ofd_commitrw+0x5b4/0xd20 [ofd] [97368.200995] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.202306] obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.203559] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97368.204883] ? flush_work+0x42/0x1d0 [97368.205872] ? internal_add_timer+0x42/0x70 [97368.206962] ? _cond_resched+0x15/0x30 [97368.207940] ? mutex_lock+0xe/0x30 [97368.208856] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97368.210227] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97368.211728] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97368.213230] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97368.214353] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97368.215425] kthread+0x134/0x150 [97368.216169] ? set_kthread_struct+0x50/0x50 [97368.217068] ret_from_fork+0x1f/0x40 [97368.217999] CPU: 0 PID: 188099 Comm: ll_ost_io00_002 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97368.220310] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97368.222354] Call Trace: [97368.223206] dump_stack+0x41/0x60 [97368.224151] warn_alloc.cold.127+0x7b/0x108 [97368.225237] __alloc_pages_slowpath+0xcb2/0xcd0 [97368.226380] ? __alloc_pages_nodemask+0x166/0x330 [97368.227558] __alloc_pages_nodemask+0x2e2/0x330 [97368.228730] kmalloc_order+0x28/0x90 [97368.229689] kmalloc_order_trace+0x1d/0xb0 [97368.230761] __kmalloc+0x203/0x250 [97368.231687] virtqueue_add+0x493/0xc70 [97368.232719] ? finish_wait+0x80/0x80 [97368.233701] virtqueue_add_sgs+0x80/0xa0 [97368.234769] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97368.236095] ? scsi_alloc_sgtables+0x84/0x1c0 [97368.237193] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97368.238496] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97368.239832] scsi_queue_rq+0x512/0xb10 [97368.240817] __blk_mq_try_issue_directly+0x163/0x200 [97368.242047] blk_mq_request_issue_directly+0x4e/0xb0 [97368.243300] blk_mq_try_issue_list_directly+0x62/0x100 [97368.244550] blk_mq_sched_insert_requests+0xa4/0xf0 [97368.245737] blk_mq_flush_plug_list+0x135/0x220 [97368.246890] blk_flush_plug_list+0xd7/0x100 [97368.247946] blk_finish_plug+0x25/0x36 [97368.248925] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97368.250330] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97368.251622] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.253115] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97368.254630] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97368.255948] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97368.257222] ofd_commitrw+0x5b4/0xd20 [ofd] [97368.258372] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.259701] obd_commitrw+0x1b6/0x370 [ptlrpc] [97368.260960] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97368.262274] ? flush_work+0x42/0x1d0 [97368.263280] ? internal_add_timer+0x42/0x70 [97368.264371] ? _cond_resched+0x15/0x30 [97368.265370] ? mutex_lock+0xe/0x30 [97368.266304] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97368.267720] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97368.269279] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97368.270835] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97368.272145] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97368.273540] kthread+0x134/0x150 [97368.274440] ? set_kthread_struct+0x50/0x50 [97368.275511] ret_from_fork+0x1f/0x40 [97369.035899] warn_alloc: 51 callbacks suppressed [97369.035902] kworker/6:1H: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97369.036029] kworker/17:1H: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97369.037318] ,cpuset= [97369.038603] ll_ost_io04_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97369.038615] CPU: 19 PID: 527216 Comm: ll_ost_io04_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.038619] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.038620] Call Trace: [97369.038624] dump_stack+0x41/0x60 [97369.038633] warn_alloc.cold.127+0x7b/0x108 [97369.038639] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.038644] __alloc_pages_nodemask+0x2e2/0x330 [97369.038647] kmalloc_order+0x28/0x90 [97369.038654] kmalloc_order_trace+0x1d/0xb0 [97369.038657] __kmalloc+0x203/0x250 [97369.038663] virtqueue_add+0x493/0xc70 [97369.038669] ? finish_wait+0x80/0x80 [97369.038674] virtqueue_add_sgs+0x80/0xa0 [97369.038679] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.038685] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.038690] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.038694] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.038698] scsi_queue_rq+0x512/0xb10 [97369.038701] __blk_mq_try_issue_directly+0x163/0x200 [97369.038708] blk_mq_request_issue_directly+0x4e/0xb0 [97369.038713] blk_mq_try_issue_list_directly+0x62/0x100 [97369.038717] blk_mq_sched_insert_requests+0xa4/0xf0 [97369.038721] blk_mq_flush_plug_list+0x135/0x220 [97369.038726] blk_flush_plug_list+0xd7/0x100 [97369.038732] blk_finish_plug+0x25/0x36 [97369.038738] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97369.038773] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97369.038806] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.038837] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.038866] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97369.038894] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97369.038918] ofd_commitrw+0x5b4/0xd20 [ofd] [97369.038938] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.039072] obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.039184] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97369.039296] ? flush_work+0x42/0x1d0 [97369.039301] ? internal_add_timer+0x42/0x70 [97369.039305] ? _cond_resched+0x15/0x30 [97369.039309] ? mutex_lock+0xe/0x30 [97369.039312] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97369.039424] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97369.039536] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97369.039642] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97369.039758] ,cpuset= [97369.039760] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97369.039867] kthread+0x134/0x150 [97369.039871] ? set_kthread_struct+0x50/0x50 [97369.039874] ret_from_fork+0x1f/0x40 [97369.039881] warn_alloc_show_mem: 5 callbacks suppressed [97369.039883] Mem-Info: [97369.039886] active_anon:27803 inactive_anon:199646 isolated_anon:0 active_file:30579170 inactive_file:3472588 isolated_file:2 unevictable:0 dirty:1511 writeback:0 slab_reclaimable:1000899 slab_unreclaimable:605121 mapped:23584 shmem:146 pagetables:5705 bounce:0 free:2125558 free_pcp:504 free_cma:0 [97369.039892] Node 0 active_anon:111212kB inactive_anon:798584kB active_file:122316680kB inactive_file:13890352kB unevictable:0kB isolated(anon):0kB isolated(file):8kB mapped:94336kB dirty:6044kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35440kB pagetables:22820kB all_unreclaimable? no [97369.039898] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97369.039904] lowmem_reserve[]: 0 913 150007 150007 150007 [97369.039910] Node 0 DMA32 free:596448kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97369.039916] lowmem_reserve[]: 0 0 149093 149093 149093 [97369.039921] Node 0 Normal free:7894520kB min:49268kB low:201940kB high:354612kB active_anon:110832kB inactive_anon:798580kB active_file:122316872kB inactive_file:13890352kB unevictable:0kB writepending:6044kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2048kB local_pcp:144kB free_cma:0kB [97369.039928] lowmem_reserve[]: 0 0 0 0 0 [97369.039932] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [97369.039946] Node 0 DMA32: 18*4kB (UM) 21*8kB (UM) 29*16kB (UME) 40*32kB (UM) 49*64kB (UMH) 46*128kB (UM) 45*256kB (UMH) 45*512kB (UME) 38*1024kB (UM) 42*2048kB (UM) 104*4096kB (UM) = 596480kB [97369.039968] Node 0 Normal: 1210632*4kB (UMEH) 354432*8kB (UMEH) 13527*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 7894416kB [97369.039984] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97369.039987] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97369.039989] 34051128 total pagecache pages [97369.039990] 1613 pages in swap cache [97369.039991] Swap cache stats: add 1610073, delete 1608483, find 83909/101242 [97369.039993] Free swap = 5326636kB [97369.039993] Total swap = 11075580kB [97369.039994] 39321462 pages RAM [97369.039995] 0 pages HighMem/MovableOnly [97369.039996] 905848 pages reserved [97369.039997] 0 pages hwpoisoned [97369.042119] / [97369.052952] ll_ost_io02_035: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97369.052966] ll_ost_io03_003: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97369.053127] / [97369.053126] ,mems_allowed=0 [97369.053132] CPU: 6 PID: 174 Comm: kworker/6:1H Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.053136] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.053146] Workqueue: kblockd blk_mq_run_work_fn [97369.053154] Call Trace: [97369.053159] dump_stack+0x41/0x60 [97369.053167] warn_alloc.cold.127+0x7b/0x108 [97369.053174] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.053178] ? __alloc_pages_nodemask+0x166/0x330 [97369.053181] __alloc_pages_nodemask+0x2e2/0x330 [97369.053185] kmalloc_order+0x28/0x90 [97369.053191] kmalloc_order_trace+0x1d/0xb0 [97369.053195] __kmalloc+0x203/0x250 [97369.053200] virtqueue_add+0x493/0xc70 [97369.053207] ? finish_wait+0x80/0x80 [97369.053212] virtqueue_add_sgs+0x80/0xa0 [97369.053217] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.053223] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.053229] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.053233] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.053237] scsi_queue_rq+0x512/0xb10 [97369.053240] blk_mq_dispatch_rq_list+0x154/0x880 [97369.053245] ? update_load_avg+0x7e/0x710 [97369.053250] __blk_mq_sched_dispatch_requests+0xb4/0x140 [97369.053255] blk_mq_sched_dispatch_requests+0x30/0x60 [97369.053258] __blk_mq_run_hw_queue+0x2b/0x70 [97369.053261] process_one_work+0x1d3/0x390 [97369.053267] ? process_one_work+0x390/0x390 [97369.053271] worker_thread+0x30/0x390 [97369.053274] ? process_one_work+0x390/0x390 [97369.053278] kthread+0x134/0x150 [97369.053281] ? set_kthread_struct+0x50/0x50 [97369.053285] ret_from_fork+0x1f/0x40 [97369.054361] ,cpuset= [97369.055542] / [97369.055542] ,mems_allowed=0 [97369.055547] CPU: 17 PID: 366 Comm: kworker/17:1H Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.055551] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.055554] Workqueue: kblockd blk_mq_run_work_fn [97369.055561] Call Trace: [97369.055565] dump_stack+0x41/0x60 [97369.055571] warn_alloc.cold.127+0x7b/0x108 [97369.055577] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.055581] ? __alloc_pages_nodemask+0x166/0x330 [97369.055585] __alloc_pages_nodemask+0x2e2/0x330 [97369.055588] kmalloc_order+0x28/0x90 [97369.055593] kmalloc_order_trace+0x1d/0xb0 [97369.055597] __kmalloc+0x203/0x250 [97369.055602] virtqueue_add+0x493/0xc70 [97369.055609] virtqueue_add_sgs+0x80/0xa0 [97369.055614] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.055620] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.055624] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.055628] scsi_queue_rq+0x512/0xb10 [97369.055632] blk_mq_dispatch_rq_list+0x154/0x880 [97369.055636] ? kvm_sched_clock_read+0xd/0x20 [97369.055640] ? sched_clock+0x5/0x10 [97369.055645] ? sched_clock_cpu+0xc/0xb0 [97369.055651] __blk_mq_sched_dispatch_requests+0xb4/0x140 [97369.055655] blk_mq_sched_dispatch_requests+0x30/0x60 [97369.055657] __blk_mq_run_hw_queue+0x2b/0x70 [97369.055661] process_one_work+0x1d3/0x390 [97369.055666] ? process_one_work+0x390/0x390 [97369.055669] worker_thread+0x30/0x390 [97369.055673] ? process_one_work+0x390/0x390 [97369.055676] kthread+0x134/0x150 [97369.055681] ? set_kthread_struct+0x50/0x50 [97369.055684] ret_from_fork+0x1f/0x40 [97369.058879] kworker/17:1H: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97369.059690] ll_ost_io04_031: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97369.059753] ,mems_allowed=0 [97369.059754] /,mems_allowed=0 [97369.059757] / [97369.059758] ,mems_allowed=0 [97369.059761] CPU: 9 PID: 527211 Comm: ll_ost_io02_035 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.059765] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.059766] Call Trace: [97369.059769] dump_stack+0x41/0x60 [97369.059776] warn_alloc.cold.127+0x7b/0x108 [97369.059782] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.059786] ? __alloc_pages_nodemask+0x166/0x330 [97369.059790] __alloc_pages_nodemask+0x2e2/0x330 [97369.059793] kmalloc_order+0x28/0x90 [97369.059799] kmalloc_order_trace+0x1d/0xb0 [97369.059803] __kmalloc+0x203/0x250 [97369.059807] ? __wake_up_common+0x7a/0x190 [97369.059812] virtqueue_add+0x493/0xc70 [97369.059817] ? finish_wait+0x80/0x80 [97369.059820] virtqueue_add_sgs+0x80/0xa0 [97369.059825] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.059830] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.059836] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.059840] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.059844] scsi_queue_rq+0x512/0xb10 [97369.059847] __blk_mq_try_issue_directly+0x163/0x200 [97369.059853] blk_mq_request_issue_directly+0x4e/0xb0 [97369.059857] blk_mq_try_issue_list_directly+0x62/0x100 [97369.059861] blk_mq_sched_insert_requests+0xa4/0xf0 [97369.059865] blk_mq_flush_plug_list+0x135/0x220 [97369.059870] blk_flush_plug_list+0xd7/0x100 [97369.059875] blk_finish_plug+0x25/0x36 [97369.059881] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97369.059919] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97369.059952] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.059980] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.060009] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97369.060037] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97369.060059] ofd_commitrw+0x5b4/0xd20 [ofd] [97369.060079] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.060217] obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.060327] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97369.060438] ? flush_work+0x42/0x1d0 [97369.060444] ? internal_add_timer+0x42/0x70 [97369.060448] ? _cond_resched+0x15/0x30 [97369.060451] ? mutex_lock+0xe/0x30 [97369.060454] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97369.060576] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97369.060684] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97369.060829] ,cpuset= [97369.060803] ? tgt_key_exit+0xc/0x20 [ptlrpc] [97369.060921] ll_ost_io04_020: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [97369.060926] ll_ost_io04_071: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97369.060927] ,mems_allowed=0 [97369.060929] ,cpuset=/,mems_allowed=0 [97369.060936] ll_ost_io03_079: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97369.060919] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97369.061041] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97369.061157] kthread+0x134/0x150 [97369.061162] ? set_kthread_struct+0x50/0x50 [97369.061165] ret_from_fork+0x1f/0x40 [97369.061171] CPU: 18 PID: 528059 Comm: ll_ost_io04_071 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.061174] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.061175] Call Trace: [97369.061178] dump_stack+0x41/0x60 [97369.061184] warn_alloc.cold.127+0x7b/0x108 [97369.061189] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.061192] ? __alloc_pages_nodemask+0x166/0x330 [97369.061195] __alloc_pages_nodemask+0x2e2/0x330 [97369.061198] kmalloc_order+0x28/0x90 [97369.061203] kmalloc_order_trace+0x1d/0xb0 [97369.061206] __kmalloc+0x203/0x250 [97369.061210] virtqueue_add+0x493/0xc70 [97369.061215] ? finish_wait+0x80/0x80 [97369.061219] virtqueue_add_sgs+0x80/0xa0 [97369.061223] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.061227] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.061232] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.061235] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.061238] scsi_queue_rq+0x512/0xb10 [97369.061258] __blk_mq_try_issue_directly+0x163/0x200 [97369.061263] blk_mq_request_issue_directly+0x4e/0xb0 [97369.061267] blk_mq_try_issue_list_directly+0x62/0x100 [97369.061270] blk_mq_sched_insert_requests+0xa4/0xf0 [97369.061274] blk_mq_flush_plug_list+0x135/0x220 [97369.061277] blk_flush_plug_list+0xd7/0x100 [97369.061282] blk_finish_plug+0x25/0x36 [97369.061287] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97369.061323] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97369.061353] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.061377] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.061403] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97369.061428] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97369.061447] ofd_commitrw+0x5b4/0xd20 [ofd] [97369.061464] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.061595] obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.061702] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97369.061808] ? flush_work+0x42/0x1d0 [97369.061812] ? internal_add_timer+0x42/0x70 [97369.061815] ? _cond_resched+0x15/0x30 [97369.061817] ? mutex_lock+0xe/0x30 [97369.061819] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97369.061932] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97369.062037] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97369.062139] ? lu_context_exit+0x12/0x70 [obdclass] [97369.062221] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97369.062319] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97369.062426] kthread+0x134/0x150 [97369.062430] ? set_kthread_struct+0x50/0x50 [97369.062433] ret_from_fork+0x1f/0x40 [97369.062443] CPU: 16 PID: 527235 Comm: ll_ost_io04_020 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.062448] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.062449] Call Trace: [97369.062451] dump_stack+0x41/0x60 [97369.062457] warn_alloc.cold.127+0x7b/0x108 [97369.062464] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.062467] ? __alloc_pages_nodemask+0x166/0x330 [97369.062471] __alloc_pages_nodemask+0x2e2/0x330 [97369.062474] kmalloc_order+0x28/0x90 [97369.062479] kmalloc_order_trace+0x1d/0xb0 [97369.062483] __kmalloc+0x203/0x250 [97369.062487] virtqueue_add+0x493/0xc70 [97369.062492] ? finish_wait+0x80/0x80 [97369.062496] virtqueue_add_sgs+0x80/0xa0 [97369.062501] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.062506] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.062512] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.062515] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.062518] scsi_queue_rq+0x512/0xb10 [97369.062521] __blk_mq_try_issue_directly+0x163/0x200 [97369.062526] blk_mq_request_issue_directly+0x4e/0xb0 [97369.062531] blk_mq_try_issue_list_directly+0x62/0x100 [97369.062535] blk_mq_sched_insert_requests+0xa4/0xf0 [97369.062539] blk_mq_flush_plug_list+0x135/0x220 [97369.062544] blk_flush_plug_list+0xd7/0x100 [97369.062550] blk_finish_plug+0x25/0x36 [97369.062555] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97369.062593] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97369.062624] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.062652] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.062684] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97369.062715] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97369.062737] ofd_commitrw+0x5b4/0xd20 [ofd] [97369.062755] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.062883] obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.062999] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97369.063114] ? flush_work+0x42/0x1d0 [97369.063119] ? internal_add_timer+0x42/0x70 [97369.063122] ? _cond_resched+0x15/0x30 [97369.063125] ? mutex_lock+0xe/0x30 [97369.063127] / [97369.063128] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97369.063226] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97369.063331] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97369.063440] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97369.063552] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97369.063660] kthread+0x134/0x150 [97369.063663] ? set_kthread_struct+0x50/0x50 [97369.063666] ret_from_fork+0x1f/0x40 [97369.063672] CPU: 19 PID: 527257 Comm: ll_ost_io04_031 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.063676] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.063678] Call Trace: [97369.063680] dump_stack+0x41/0x60 [97369.063687] warn_alloc.cold.127+0x7b/0x108 [97369.063692] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.063696] ? __alloc_pages_nodemask+0x166/0x330 [97369.063699] __alloc_pages_nodemask+0x2e2/0x330 [97369.063702] kmalloc_order+0x28/0x90 [97369.063707] kmalloc_order_trace+0x1d/0xb0 [97369.063711] __kmalloc+0x203/0x250 [97369.063716] virtqueue_add+0x493/0xc70 [97369.063721] ? finish_wait+0x80/0x80 [97369.063725] virtqueue_add_sgs+0x80/0xa0 [97369.063730] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.063735] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.063740] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.063744] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.063748] scsi_queue_rq+0x512/0xb10 [97369.063751] __blk_mq_try_issue_directly+0x163/0x200 [97369.063757] blk_mq_request_issue_directly+0x4e/0xb0 [97369.063761] blk_mq_try_issue_list_directly+0x62/0x100 [97369.063765] blk_mq_sched_insert_requests+0xa4/0xf0 [97369.063769] blk_mq_flush_plug_list+0x135/0x220 [97369.063774] blk_flush_plug_list+0xd7/0x100 [97369.063779] blk_finish_plug+0x25/0x36 [97369.063783] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97369.063818] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [97369.063851] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.063876] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.063902] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97369.063930] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97369.063953] ofd_commitrw+0x5b4/0xd20 [ofd] [97369.063969] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.064081] obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.064192] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97369.065567] ,mems_allowed=0 [97369.067118] ? flush_work+0x42/0x1d0 [97369.069399] ? internal_add_timer+0x42/0x70 [97369.392474] ? _cond_resched+0x15/0x30 [97369.393181] ? mutex_lock+0xe/0x30 [97369.393831] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97369.394804] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97369.395882] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97369.396993] ? finish_wait+0x80/0x80 [97369.397706] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97369.398633] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97369.399623] kthread+0x134/0x150 [97369.400277] ? set_kthread_struct+0x50/0x50 [97369.401026] ret_from_fork+0x1f/0x40 [97369.401681] CPU: 12 PID: 527552 Comm: ll_ost_io03_079 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.404297] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.406207] Call Trace: [97369.406823] dump_stack+0x41/0x60 [97369.407658] warn_alloc.cold.127+0x7b/0x108 [97369.408744] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.409948] __alloc_pages_nodemask+0x2e2/0x330 [97369.410876] kmalloc_order+0x28/0x90 [97369.411881] kmalloc_order_trace+0x1d/0xb0 [97369.412878] __kmalloc+0x203/0x250 [97369.413810] virtqueue_add+0x493/0xc70 [97369.414900] ? finish_wait+0x80/0x80 [97369.415894] virtqueue_add_sgs+0x80/0xa0 [97369.417264] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.418544] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.419516] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.420743] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.422107] scsi_queue_rq+0x512/0xb10 [97369.423050] __blk_mq_try_issue_directly+0x163/0x200 [97369.423982] blk_mq_request_issue_directly+0x4e/0xb0 [97369.424897] blk_mq_try_issue_list_directly+0x62/0x100 [97369.425855] blk_mq_sched_insert_requests+0xa4/0xf0 [97369.426909] blk_mq_flush_plug_list+0x135/0x220 [97369.428025] blk_flush_plug_list+0xd7/0x100 [97369.429072] blk_finish_plug+0x25/0x36 [97369.430046] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97369.431439] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97369.432726] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.434223] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.435682] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97369.436774] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97369.437768] ofd_commitrw+0x5b4/0xd20 [ofd] [97369.438858] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.440210] obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.441503] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97369.442829] ? flush_work+0x42/0x1d0 [97369.443684] ? internal_add_timer+0x42/0x70 [97369.444586] ? _cond_resched+0x15/0x30 [97369.445576] ? mutex_lock+0xe/0x30 [97369.446516] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97369.447814] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97369.449328] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97369.450900] ? finish_wait+0x80/0x80 [97369.451870] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97369.453137] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97369.454466] kthread+0x134/0x150 [97369.455276] ? set_kthread_struct+0x50/0x50 [97369.456213] ret_from_fork+0x1f/0x40 [97369.457171] CPU: 17 PID: 366 Comm: kworker/17:1H Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.459194] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.460683] Workqueue: kblockd blk_mq_run_work_fn [97369.461543] Call Trace: [97369.462094] dump_stack+0x41/0x60 [97369.462743] warn_alloc.cold.127+0x7b/0x108 [97369.463519] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.464337] ? __alloc_pages_nodemask+0x166/0x330 [97369.465180] __alloc_pages_nodemask+0x2e2/0x330 [97369.466017] kmalloc_order+0x28/0x90 [97369.466712] kmalloc_order_trace+0x1d/0xb0 [97369.467466] __kmalloc+0x203/0x250 [97369.468129] virtqueue_add+0x493/0xc70 [97369.468851] virtqueue_add_sgs+0x80/0xa0 [97369.469634] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.470581] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.471463] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.472439] scsi_queue_rq+0x512/0xb10 [97369.473160] blk_mq_dispatch_rq_list+0x154/0x880 [97369.473979] ? kvm_sched_clock_read+0xd/0x20 [97369.474745] ? sched_clock+0x5/0x10 [97369.475434] ? sched_clock_cpu+0xc/0xb0 [97369.476188] __blk_mq_sched_dispatch_requests+0xb4/0x140 [97369.477116] blk_mq_sched_dispatch_requests+0x30/0x60 [97369.478084] __blk_mq_run_hw_queue+0x2b/0x70 [97369.478882] process_one_work+0x1d3/0x390 [97369.479619] ? process_one_work+0x390/0x390 [97369.480393] worker_thread+0x30/0x390 [97369.481099] ? process_one_work+0x390/0x390 [97369.481868] kthread+0x134/0x150 [97369.482494] ? set_kthread_struct+0x50/0x50 [97369.483278] ret_from_fork+0x1f/0x40 [97369.483965] CPU: 14 PID: 527010 Comm: ll_ost_io03_003 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97369.486819] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97369.488961] Call Trace: [97369.489798] dump_stack+0x41/0x60 [97369.490813] warn_alloc.cold.127+0x7b/0x108 [97369.491937] __alloc_pages_slowpath+0xcb2/0xcd0 [97369.493131] ? __alloc_pages_nodemask+0x166/0x330 [97369.494329] __alloc_pages_nodemask+0x2e2/0x330 [97369.495533] kmalloc_order+0x28/0x90 [97369.496557] kmalloc_order_trace+0x1d/0xb0 [97369.497683] __kmalloc+0x203/0x250 [97369.498645] virtqueue_add+0x493/0xc70 [97369.499699] ? finish_wait+0x80/0x80 [97369.500683] virtqueue_add_sgs+0x80/0xa0 [97369.501792] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97369.503150] ? scsi_alloc_sgtables+0x84/0x1c0 [97369.504295] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97369.505543] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97369.506909] scsi_queue_rq+0x512/0xb10 [97369.507915] __blk_mq_try_issue_directly+0x163/0x200 [97369.509123] blk_mq_request_issue_directly+0x4e/0xb0 [97369.510338] blk_mq_try_issue_list_directly+0x62/0x100 [97369.511583] blk_mq_sched_insert_requests+0xa4/0xf0 [97369.512847] blk_mq_flush_plug_list+0x135/0x220 [97369.514039] blk_flush_plug_list+0xd7/0x100 [97369.515182] blk_finish_plug+0x25/0x36 [97369.516212] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97369.517714] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97369.519096] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.520702] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97369.522273] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97369.523636] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97369.524925] ofd_commitrw+0x5b4/0xd20 [ofd] [97369.526104] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.527486] obd_commitrw+0x1b6/0x370 [ptlrpc] [97369.528840] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97369.530260] ? flush_work+0x42/0x1d0 [97369.531345] ? internal_add_timer+0x42/0x70 [97369.532494] ? _cond_resched+0x15/0x30 [97369.533583] ? mutex_lock+0xe/0x30 [97369.534553] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97369.536030] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97369.537620] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97369.539212] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97369.540566] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97369.542036] kthread+0x134/0x150 [97369.543004] ? set_kthread_struct+0x50/0x50 [97369.544158] ret_from_fork+0x1f/0x40 [97374.861226] warn_alloc: 532 callbacks suppressed [97374.861230] ll_ost_io00_080: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97374.861251] ll_ost_io04_031: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97374.861695] ll_ost_io00_081: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97374.861708] CPU: 1 PID: 527786 Comm: ll_ost_io00_081 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97374.861712] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97374.861713] Call Trace: [97374.861719] dump_stack+0x41/0x60 [97374.861728] warn_alloc.cold.127+0x7b/0x108 [97374.861736] __alloc_pages_slowpath+0xcb2/0xcd0 [97374.861740] __alloc_pages_nodemask+0x2e2/0x330 [97374.861744] kmalloc_order+0x28/0x90 [97374.861750] kmalloc_order_trace+0x1d/0xb0 [97374.861754] __kmalloc+0x203/0x250 [97374.861760] virtqueue_add+0x493/0xc70 [97374.861767] ? finish_wait+0x80/0x80 [97374.861772] virtqueue_add_sgs+0x80/0xa0 [97374.861777] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97374.861783] ? scsi_alloc_sgtables+0x84/0x1c0 [97374.861790] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97374.861794] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97374.861797] scsi_queue_rq+0x512/0xb10 [97374.861801] blk_mq_dispatch_rq_list+0x154/0x880 [97374.861809] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [97374.861813] __blk_mq_sched_dispatch_requests+0x110/0x140 [97374.861817] blk_mq_sched_dispatch_requests+0x30/0x60 [97374.861819] __blk_mq_run_hw_queue+0x2b/0x70 [97374.861823] __blk_mq_delay_run_hw_queue+0x141/0x160 [97374.861827] blk_mq_sched_insert_requests+0x6c/0xf0 [97374.861830] blk_mq_flush_plug_list+0x135/0x220 [97374.861835] blk_flush_plug_list+0xd7/0x100 [97374.861841] blk_finish_plug+0x25/0x36 [97374.861847] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97374.861885] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97374.861918] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97374.861950] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97374.861979] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97374.862007] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97374.862029] ofd_commitrw+0x5b4/0xd20 [ofd] [97374.862049] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97374.862182] obd_commitrw+0x1b6/0x370 [ptlrpc] [97374.862294] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97374.862410] ? flush_work+0x42/0x1d0 [97374.862417] ? internal_add_timer+0x42/0x70 [97374.862421] ? _cond_resched+0x15/0x30 [97374.862424] ? mutex_lock+0xe/0x30 [97374.862428] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97374.862540] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97374.862669] ,cpuset= [97374.862650] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97374.862775] ? finish_wait+0x80/0x80 [97374.862779] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97374.862911] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97374.863026] kthread+0x134/0x150 [97374.863030] ? set_kthread_struct+0x50/0x50 [97374.863034] ret_from_fork+0x1f/0x40 [97374.863040] warn_alloc_show_mem: 9 callbacks suppressed [97374.863042] Mem-Info: [97374.863044] active_anon:27804 inactive_anon:199747 isolated_anon:0 active_file:30557904 inactive_file:3466335 isolated_file:0 unevictable:0 dirty:2136 writeback:0 slab_reclaimable:1000863 slab_unreclaimable:605099 mapped:23663 shmem:146 pagetables:5705 bounce:0 free:2152458 free_pcp:404 free_cma:0 [97374.863051] Node 0 active_anon:111216kB inactive_anon:798988kB active_file:122231616kB inactive_file:13865340kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:94652kB dirty:8544kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35440kB pagetables:22820kB all_unreclaimable? no [97374.863058] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97374.863065] lowmem_reserve[]: 0 913 150007 150007 150007 [97374.863071] Node 0 DMA32 free:596496kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97374.863077] lowmem_reserve[]: 0 0 149093 149093 149093 [97374.863082] Node 0 Normal free:8002072kB min:49268kB low:201940kB high:354612kB active_anon:110836kB inactive_anon:798984kB active_file:122231776kB inactive_file:13865340kB unevictable:0kB writepending:8544kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1624kB local_pcp:20kB free_cma:0kB [97374.863089] lowmem_reserve[]: 0 0 0 0 0 [97374.863093] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [97374.863108] Node 0 DMA32: 18*4kB (UM) 21*8kB (UM) 30*16kB (UME) 40*32kB (UM) 46*64kB (UMH) 47*128kB (UMH) 45*256kB (UMH) 45*512kB (UME) 38*1024kB (UM) 42*2048kB (UM) 104*4096kB (UM) = 596432kB [97374.863129] Node 0 Normal: 890836*4kB (UMEH) 507841*8kB (UMEH) 23443*16kB (UMEH) 1*32kB (U) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 8001192kB [97374.863145] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97374.863148] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97374.863150] 34023660 total pagecache pages [97374.863151] 1603 pages in swap cache [97374.863152] Swap cache stats: add 1610446, delete 1608866, find 84006/101353 [97374.863154] Free swap = 5326116kB [97374.863155] Total swap = 11075580kB [97374.863156] 39321462 pages RAM [97374.863156] 0 pages HighMem/MovableOnly [97374.863157] 905848 pages reserved [97374.863158] 0 pages hwpoisoned [97374.864871] ,cpuset= [97374.867407] /,mems_allowed=0 [97374.986025] / [97374.986030] CPU: 3 PID: 527785 Comm: ll_ost_io00_080 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97374.987026] ,mems_allowed=0 [97374.987722] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97374.991323] Call Trace: [97374.996127] dump_stack+0x41/0x60 [97374.997166] warn_alloc.cold.127+0x7b/0x108 [97374.998365] __alloc_pages_slowpath+0xcb2/0xcd0 [97374.999634] ? __alloc_pages_nodemask+0x166/0x330 [97375.000940] __alloc_pages_nodemask+0x2e2/0x330 [97375.002225] kmalloc_order+0x28/0x90 [97375.003303] kmalloc_order_trace+0x1d/0xb0 [97375.004497] __kmalloc+0x203/0x250 [97375.005544] virtqueue_add+0x493/0xc70 [97375.006632] ? finish_wait+0x80/0x80 [97375.007694] virtqueue_add_sgs+0x80/0xa0 [97375.008811] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.010220] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.011428] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.012741] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.014216] scsi_queue_rq+0x512/0xb10 [97375.015277] blk_mq_dispatch_rq_list+0x154/0x880 [97375.016501] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [97375.017704] __blk_mq_sched_dispatch_requests+0x110/0x140 [97375.019042] blk_mq_sched_dispatch_requests+0x30/0x60 [97375.020254] __blk_mq_run_hw_queue+0x2b/0x70 [97375.021095] __blk_mq_delay_run_hw_queue+0x141/0x160 [97375.022015] blk_mq_sched_insert_requests+0x6c/0xf0 [97375.022904] blk_mq_flush_plug_list+0x135/0x220 [97375.023756] blk_flush_plug_list+0xd7/0x100 [97375.024716] blk_finish_plug+0x25/0x36 [97375.025711] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.027177] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [97375.028495] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.029972] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.031453] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.032736] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.033937] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.035010] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.036278] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.037506] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.038789] ? flush_work+0x42/0x1d0 [97375.039728] ? internal_add_timer+0x42/0x70 [97375.040780] ? _cond_resched+0x15/0x30 [97375.041742] ? mutex_lock+0xe/0x30 [97375.042645] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.044045] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.045580] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.047095] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.048335] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.049713] kthread+0x134/0x150 [97375.050600] ? set_kthread_struct+0x50/0x50 [97375.051638] ret_from_fork+0x1f/0x40 [97375.052573] CPU: 18 PID: 527257 Comm: ll_ost_io04_031 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.055378] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.057469] Call Trace: [97375.058241] dump_stack+0x41/0x60 [97375.059162] warn_alloc.cold.127+0x7b/0x108 [97375.060247] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.061394] ? __alloc_pages_nodemask+0x166/0x330 [97375.062595] __alloc_pages_nodemask+0x2e2/0x330 [97375.063741] kmalloc_order+0x28/0x90 [97375.064699] kmalloc_order_trace+0x1d/0xb0 [97375.065832] __kmalloc+0x203/0x250 [97375.066805] virtqueue_add+0x493/0xc70 [97375.067795] virtqueue_add_sgs+0x80/0xa0 [97375.068809] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.070141] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.071287] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.072540] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.073917] scsi_queue_rq+0x512/0xb10 [97375.074950] blk_mq_dispatch_rq_list+0x154/0x880 [97375.076133] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [97375.077314] __blk_mq_sched_dispatch_requests+0x110/0x140 [97375.078650] blk_mq_sched_dispatch_requests+0x30/0x60 [97375.079985] __blk_mq_run_hw_queue+0x2b/0x70 [97375.081136] __blk_mq_delay_run_hw_queue+0x141/0x160 [97375.082439] blk_mq_sched_insert_requests+0x6c/0xf0 [97375.083709] blk_mq_flush_plug_list+0x135/0x220 [97375.084887] blk_flush_plug_list+0xd7/0x100 [97375.086013] blk_finish_plug+0x25/0x36 [97375.087049] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.088508] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97375.089869] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.091439] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.092967] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.094321] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.095600] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.096748] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.098118] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.099478] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.100867] ? flush_work+0x42/0x1d0 [97375.101877] ? internal_add_timer+0x42/0x70 [97375.103029] ? _cond_resched+0x15/0x30 [97375.104066] ? mutex_lock+0xe/0x30 [97375.105048] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.106511] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.108114] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.109696] ? finish_wait+0x80/0x80 [97375.110709] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.112011] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.113460] kthread+0x134/0x150 [97375.114394] ? set_kthread_struct+0x50/0x50 [97375.115505] ret_from_fork+0x1f/0x40 [97375.397115] ll_ost_io03_066: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97375.400061] CPU: 15 PID: 527451 Comm: ll_ost_io03_066 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.402999] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.405079] ll_ost_io03_067: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97375.405106] ll_ost_io04_049: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97375.405172] Call Trace: [97375.407146] ,cpuset= [97375.409647] dump_stack+0x41/0x60 [97375.410347] ll_ost_io01_058: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97375.410411] / [97375.411111] warn_alloc.cold.127+0x7b/0x108 [97375.411992] ,mems_allowed=0 [97375.412664] ll_ost_io00_016: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97375.414317] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.415852] ? __alloc_pages_nodemask+0x166/0x330 [97375.421580] __alloc_pages_nodemask+0x2e2/0x330 [97375.422614] kmalloc_order+0x28/0x90 [97375.423170] ll_ost_io03_080: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97375.423470] kmalloc_order_trace+0x1d/0xb0 [97375.427157] __kmalloc+0x203/0x250 [97375.428731] virtqueue_add+0x493/0xc70 [97375.429624] ? finish_wait+0x80/0x80 [97375.430529] virtqueue_add_sgs+0x80/0xa0 [97375.431469] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.432611] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.433574] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.434651] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.435823] scsi_queue_rq+0x512/0xb10 [97375.436698] __blk_mq_try_issue_directly+0x163/0x200 [97375.437755] blk_mq_request_issue_directly+0x4e/0xb0 [97375.438823] blk_mq_try_issue_list_directly+0x62/0x100 [97375.439913] blk_mq_sched_insert_requests+0xa4/0xf0 [97375.440258] ll_ost_io02_041: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [97375.440964] blk_mq_flush_plug_list+0x135/0x220 [97375.440969] blk_flush_plug_list+0xd7/0x100 [97375.443633] ,mems_allowed=0 [97375.444701] blk_finish_plug+0x25/0x36 [97375.446770] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.449777] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97375.450934] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.452263] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.453575] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.454726] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.455792] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.456779] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.457941] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.459081] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.460238] ? set_next_entity+0xb5/0x1e0 [97375.461152] ? internal_add_timer+0x42/0x70 [97375.462086] ? _cond_resched+0x15/0x30 [97375.462954] ? mutex_lock+0xe/0x30 [97375.463751] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.464925] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.466218] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.467498] ? finish_wait+0x80/0x80 [97375.468304] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.469339] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.470481] kthread+0x134/0x150 [97375.471232] ? set_kthread_struct+0x50/0x50 [97375.472121] ret_from_fork+0x1f/0x40 [97375.472926] CPU: 12 PID: 527553 Comm: ll_ost_io03_080 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.475822] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.477636] Call Trace: [97375.478222] dump_stack+0x41/0x60 [97375.479036] warn_alloc.cold.127+0x7b/0x108 [97375.480154] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.481119] ? virtqueue_add+0x959/0xc70 [97375.481926] __alloc_pages_nodemask+0x2e2/0x330 [97375.483001] kmalloc_order+0x28/0x90 [97375.483994] kmalloc_order_trace+0x1d/0xb0 [97375.485127] __kmalloc+0x203/0x250 [97375.486242] virtqueue_add+0x493/0xc70 [97375.487290] ? finish_wait+0x80/0x80 [97375.488267] virtqueue_add_sgs+0x80/0xa0 [97375.489301] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.490601] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.491698] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.492759] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.493773] scsi_queue_rq+0x512/0xb10 [97375.494536] __blk_mq_try_issue_directly+0x163/0x200 [97375.495473] blk_mq_request_issue_directly+0x4e/0xb0 [97375.496541] blk_mq_try_issue_list_directly+0x62/0x100 [97375.497757] blk_mq_sched_insert_requests+0xa4/0xf0 [97375.498806] blk_mq_flush_plug_list+0x135/0x220 [97375.499897] blk_flush_plug_list+0xd7/0x100 [97375.500729] blk_finish_plug+0x25/0x36 [97375.501562] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.502850] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97375.504144] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.505654] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.507115] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.508384] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.509572] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.510635] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.511884] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.513102] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.514279] ? flush_work+0x42/0x1d0 [97375.515136] ? internal_add_timer+0x42/0x70 [97375.516106] ? _cond_resched+0x15/0x30 [97375.516924] ? mutex_lock+0xe/0x30 [97375.517809] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.519164] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.520660] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.522004] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.523229] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.524566] kthread+0x134/0x150 [97375.525428] ? set_kthread_struct+0x50/0x50 [97375.526477] ret_from_fork+0x1f/0x40 [97375.527415] CPU: 18 PID: 527686 Comm: ll_ost_io04_049 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.530170] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.531682] Call Trace: [97375.532207] dump_stack+0x41/0x60 [97375.532839] warn_alloc.cold.127+0x7b/0x108 [97375.533589] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.534394] ? __alloc_pages_nodemask+0x166/0x330 [97375.535223] __alloc_pages_nodemask+0x2e2/0x330 [97375.536230] kmalloc_order+0x28/0x90 [97375.537051] kmalloc_order_trace+0x1d/0xb0 [97375.538098] __kmalloc+0x203/0x250 [97375.538986] virtqueue_add+0x493/0xc70 [97375.539866] virtqueue_add_sgs+0x80/0xa0 [97375.540706] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.541641] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.542435] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.543313] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.544276] scsi_queue_rq+0x512/0xb10 [97375.544979] __blk_mq_try_issue_directly+0x163/0x200 [97375.545844] blk_mq_request_issue_directly+0x4e/0xb0 [97375.546716] blk_mq_try_issue_list_directly+0x62/0x100 [97375.547622] blk_mq_sched_insert_requests+0xa4/0xf0 [97375.548484] blk_mq_flush_plug_list+0x135/0x220 [97375.549352] blk_flush_plug_list+0xd7/0x100 [97375.550253] blk_finish_plug+0x25/0x36 [97375.551216] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.552370] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.553486] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.554566] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.555510] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.556381] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.557173] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.558126] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.559031] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.559974] ? flush_work+0x42/0x1d0 [97375.560684] ? internal_add_timer+0x42/0x70 [97375.561468] ? _cond_resched+0x15/0x30 [97375.562196] ? mutex_lock+0xe/0x30 [97375.562879] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.563930] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.565263] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.566606] ? finish_wait+0x80/0x80 [97375.567568] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.568830] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.570073] kthread+0x134/0x150 [97375.570767] ? set_kthread_struct+0x50/0x50 [97375.571795] ret_from_fork+0x1f/0x40 [97375.572750] CPU: 0 PID: 527182 Comm: ll_ost_io00_016 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.575047] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.576507] Call Trace: [97375.577059] dump_stack+0x41/0x60 [97375.577709] warn_alloc.cold.127+0x7b/0x108 [97375.578507] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.579349] ? __alloc_pages_nodemask+0x166/0x330 [97375.580200] __alloc_pages_nodemask+0x2e2/0x330 [97375.581018] kmalloc_order+0x28/0x90 [97375.581705] kmalloc_order_trace+0x1d/0xb0 [97375.582467] __kmalloc+0x203/0x250 [97375.583118] virtqueue_add+0x493/0xc70 [97375.583812] ? finish_wait+0x80/0x80 [97375.584482] virtqueue_add_sgs+0x80/0xa0 [97375.585217] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.586154] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.586945] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.587831] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.588789] scsi_queue_rq+0x512/0xb10 [97375.589493] __blk_mq_try_issue_directly+0x163/0x200 [97375.590355] blk_mq_request_issue_directly+0x4e/0xb0 [97375.591224] blk_mq_try_issue_list_directly+0x62/0x100 [97375.592127] blk_mq_sched_insert_requests+0xa4/0xf0 [97375.592991] blk_mq_flush_plug_list+0x135/0x220 [97375.593795] blk_flush_plug_list+0xd7/0x100 [97375.594573] blk_finish_plug+0x25/0x36 [97375.595304] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.596322] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97375.597264] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.598336] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.599381] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.600304] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.601165] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.601926] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.602817] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.603699] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.604600] ? flush_work+0x42/0x1d0 [97375.605267] ? internal_add_timer+0x42/0x70 [97375.606032] ? _cond_resched+0x15/0x30 [97375.606732] ? mutex_lock+0xe/0x30 [97375.607375] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.608356] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.609417] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.610481] ? tgt_mask_cksum_types+0x70/0x70 [ptlrpc] [97375.611443] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.612327] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.613314] kthread+0x134/0x150 [97375.613969] ? set_kthread_struct+0x50/0x50 [97375.614735] ret_from_fork+0x1f/0x40 [97375.615407] CPU: 8 PID: 527246 Comm: ll_ost_io02_041 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.618158] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.620092] Call Trace: [97375.620740] dump_stack+0x41/0x60 [97375.621660] warn_alloc.cold.127+0x7b/0x108 [97375.622734] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.623852] ? __alloc_pages_nodemask+0x166/0x330 [97375.625034] __alloc_pages_nodemask+0x2e2/0x330 [97375.626196] kmalloc_order+0x28/0x90 [97375.627226] kmalloc_order_trace+0x1d/0xb0 [97375.628216] __kmalloc+0x203/0x250 [97375.629103] virtqueue_add+0x493/0xc70 [97375.630123] virtqueue_add_sgs+0x80/0xa0 [97375.631225] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.632632] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.633782] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.635079] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.636507] scsi_queue_rq+0x512/0xb10 [97375.637542] __blk_mq_try_issue_directly+0x163/0x200 [97375.638841] blk_mq_request_issue_directly+0x4e/0xb0 [97375.640171] blk_mq_try_issue_list_directly+0x62/0x100 [97375.641521] blk_mq_sched_insert_requests+0xa4/0xf0 [97375.642809] blk_mq_flush_plug_list+0x135/0x220 [97375.644032] blk_flush_plug_list+0xd7/0x100 [97375.645190] blk_finish_plug+0x25/0x36 [97375.646239] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.647748] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97375.649116] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.650761] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.652349] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.653748] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.655080] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.656244] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.657653] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.658967] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.660306] ? flush_work+0x42/0x1d0 [97375.661324] ? internal_add_timer+0x42/0x70 [97375.662435] ? _cond_resched+0x15/0x30 [97375.663490] ? mutex_lock+0xe/0x30 [97375.664436] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.665840] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.667410] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.669022] ? osd_it_ea_store+0x20/0x20 [osd_ldiskfs] [97375.670355] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.671605] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.672969] kthread+0x134/0x150 [97375.673868] ? set_kthread_struct+0x50/0x50 [97375.674945] ret_from_fork+0x1f/0x40 [97375.675934] CPU: 5 PID: 527162 Comm: ll_ost_io01_058 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.678588] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.680647] Call Trace: [97375.681394] dump_stack+0x41/0x60 [97375.682285] warn_alloc.cold.127+0x7b/0x108 [97375.683342] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.684350] ? __alloc_pages_nodemask+0x166/0x330 [97375.685541] __alloc_pages_nodemask+0x2e2/0x330 [97375.686691] kmalloc_order+0x28/0x90 [97375.687675] kmalloc_order_trace+0x1d/0xb0 [97375.688752] __kmalloc+0x203/0x250 [97375.689680] virtqueue_add+0x493/0xc70 [97375.690660] virtqueue_add_sgs+0x80/0xa0 [97375.691670] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.692980] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.694094] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.695341] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.696697] scsi_queue_rq+0x512/0xb10 [97375.697685] __blk_mq_try_issue_directly+0x163/0x200 [97375.698917] blk_mq_request_issue_directly+0x4e/0xb0 [97375.700139] blk_mq_try_issue_list_directly+0x62/0x100 [97375.701409] blk_mq_sched_insert_requests+0xa4/0xf0 [97375.702601] blk_mq_flush_plug_list+0x135/0x220 [97375.703742] blk_flush_plug_list+0xd7/0x100 [97375.704809] blk_finish_plug+0x25/0x36 [97375.705789] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.707168] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97375.708497] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.710020] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.711528] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.712735] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.713866] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.714943] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.716043] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.717012] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.718220] ? flush_work+0x42/0x1d0 [97375.719119] ? internal_add_timer+0x42/0x70 [97375.720057] ? _cond_resched+0x15/0x30 [97375.720976] ? mutex_lock+0xe/0x30 [97375.721698] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.722798] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.724262] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.725690] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.726759] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.728140] kthread+0x134/0x150 [97375.729011] ? set_kthread_struct+0x50/0x50 [97375.729843] ret_from_fork+0x1f/0x40 [97375.730750] CPU: 14 PID: 527452 Comm: ll_ost_io03_067 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97375.733617] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97375.736350] Call Trace: [97375.737198] dump_stack+0x41/0x60 [97375.738231] warn_alloc.cold.127+0x7b/0x108 [97375.739394] __alloc_pages_slowpath+0xcb2/0xcd0 [97375.740622] ? __alloc_pages_nodemask+0x166/0x330 [97375.741888] __alloc_pages_nodemask+0x2e2/0x330 [97375.743115] kmalloc_order+0x28/0x90 [97375.744168] kmalloc_order_trace+0x1d/0xb0 [97375.745351] __kmalloc+0x203/0x250 [97375.746399] virtqueue_add+0x493/0xc70 [97375.747497] ? finish_wait+0x80/0x80 [97375.748535] virtqueue_add_sgs+0x80/0xa0 [97375.749629] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97375.751142] ? scsi_alloc_sgtables+0x84/0x1c0 [97375.752374] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97375.753680] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97375.755169] scsi_queue_rq+0x512/0xb10 [97375.756233] __blk_mq_try_issue_directly+0x163/0x200 [97375.757509] blk_mq_request_issue_directly+0x4e/0xb0 [97375.758811] blk_mq_try_issue_list_directly+0x62/0x100 [97375.760109] blk_mq_sched_insert_requests+0xa4/0xf0 [97375.761368] blk_mq_flush_plug_list+0x135/0x220 [97375.762561] blk_flush_plug_list+0xd7/0x100 [97375.763673] blk_finish_plug+0x25/0x36 [97375.764696] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97375.766179] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97375.767550] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.769179] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97375.770758] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97375.772156] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97375.773510] ofd_commitrw+0x5b4/0xd20 [ofd] [97375.774728] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.776120] obd_commitrw+0x1b6/0x370 [ptlrpc] [97375.777443] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97375.778829] ? flush_work+0x42/0x1d0 [97375.779857] ? internal_add_timer+0x42/0x70 [97375.780992] ? _cond_resched+0x15/0x30 [97375.782039] ? mutex_lock+0xe/0x30 [97375.783003] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97375.784429] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97375.786077] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97375.787619] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97375.788910] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97375.790307] kthread+0x134/0x150 [97375.791230] ? set_kthread_struct+0x50/0x50 [97375.792346] ret_from_fork+0x1f/0x40 [97442.990422] warn_alloc: 2 callbacks suppressed [97442.990426] ll_ost_io04_052: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97442.990494] ll_ost_io04_027: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97442.990505] ll_ost_io00_008: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97442.990517] CPU: 0 PID: 527126 Comm: ll_ost_io00_008 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97442.990521] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97442.990523] Call Trace: [97442.990528] dump_stack+0x41/0x60 [97442.990537] warn_alloc.cold.127+0x7b/0x108 [97442.990543] __alloc_pages_slowpath+0xcb2/0xcd0 [97442.990548] ? __alloc_pages_nodemask+0x166/0x330 [97442.990554] __alloc_pages_nodemask+0x2e2/0x330 [97442.990558] kmalloc_order+0x28/0x90 [97442.990565] kmalloc_order_trace+0x1d/0xb0 [97442.990573] __kmalloc+0x203/0x250 [97442.990578] virtqueue_add+0x493/0xc70 [97442.990589] ? finish_wait+0x80/0x80 [97442.990598] virtqueue_add_sgs+0x80/0xa0 [97442.990603] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97442.990615] ? scsi_alloc_sgtables+0x84/0x1c0 [97442.990624] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97442.990628] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97442.990636] scsi_queue_rq+0x512/0xb10 [97442.990639] __blk_mq_try_issue_directly+0x163/0x200 [97442.990650] blk_mq_request_issue_directly+0x4e/0xb0 [97442.990654] blk_mq_try_issue_list_directly+0x62/0x100 [97442.990658] blk_mq_sched_insert_requests+0xa4/0xf0 [97442.990664] blk_mq_flush_plug_list+0x135/0x220 [97442.990667] blk_flush_plug_list+0xd7/0x100 [97442.990671] blk_finish_plug+0x25/0x36 [97442.990680] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97442.990718] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97442.990743] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97442.990766] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97442.990791] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97442.990809] ofd_commitrw+0x5b4/0xd20 [ofd] [97442.990828] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97442.990951] obd_commitrw+0x1b6/0x370 [ptlrpc] [97442.991045] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97442.991139] ? flush_work+0x42/0x1d0 [97442.991143] ? internal_add_timer+0x42/0x70 [97442.991146] ? _cond_resched+0x15/0x30 [97442.991152] ? mutex_lock+0xe/0x30 [97442.991154] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97442.991259] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97442.991347] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97442.991416] ? finish_wait+0x80/0x80 [97442.991419] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97442.991491] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97442.991562] kthread+0x134/0x150 [97442.991566] ? set_kthread_struct+0x50/0x50 [97442.991568] ret_from_fork+0x1f/0x40 [97442.991573] warn_alloc_show_mem: 9 callbacks suppressed [97442.991574] Mem-Info: [97442.991576] active_anon:27878 inactive_anon:199953 isolated_anon:0 active_file:30707382 inactive_file:3721318 isolated_file:1 unevictable:0 dirty:1399 writeback:0 slab_reclaimable:1000513 slab_unreclaimable:604537 mapped:23660 shmem:146 pagetables:5708 bounce:0 free:1750284 free_pcp:401 free_cma:0 [97442.991580] Node 0 active_anon:111512kB inactive_anon:799812kB active_file:122829528kB inactive_file:14885272kB unevictable:0kB isolated(anon):0kB isolated(file):4kB mapped:94640kB dirty:5596kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35488kB pagetables:22832kB all_unreclaimable? no [97442.991585] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97442.991589] lowmem_reserve[]: 0 913 150007 150007 150007 [97442.991593] Node 0 DMA32 free:596480kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97442.991597] lowmem_reserve[]: 0 0 149093 149093 149093 [97442.991600] Node 0 Normal free:6393392kB min:49268kB low:201940kB high:354612kB active_anon:111132kB inactive_anon:799808kB active_file:122829680kB inactive_file:14885144kB unevictable:0kB writepending:5596kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1600kB local_pcp:40kB free_cma:0kB [97442.991605] lowmem_reserve[]: 0 0 0 0 0 [97442.991607] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [97442.991618] Node 0 DMA32: 18*4kB (UM) 21*8kB (UM) 39*16kB (UME) 39*32kB (UM) 44*64kB (UM) 45*128kB (UM) 46*256kB (UMH) 45*512kB (UME) 38*1024kB (UM) 42*2048kB (UM) 104*4096kB (UM) = 596416kB [97442.991631] Node 0 Normal: 1093060*4kB (UMEH) [97442.991633] ,cpuset= [97442.991634] 242510*8kB (UMEH) 4986*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 6392096kB [97442.991642] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97442.991644] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97442.991645] 34428267 total pagecache pages [97442.991646] 1656 pages in swap cache [97442.991647] Swap cache stats: add 1612127, delete 1610494, find 84351/101785 [97442.991648] Free swap = 5322268kB [97442.991649] Total swap = 11075580kB [97442.991649] 39321462 pages RAM [97442.991650] 0 pages HighMem/MovableOnly [97442.991650] 905848 pages reserved [97442.991651] 0 pages hwpoisoned [97442.993712] ,cpuset= [97442.995746] / [97443.000140] ll_ost_io02_036: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97443.000491] / [97443.000491] ,mems_allowed=0 [97443.000493] ,mems_allowed=0 [97443.000496] CPU: 16 PID: 527242 Comm: ll_ost_io04_027 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97443.000500] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97443.000501] Call Trace: [97443.000506] dump_stack+0x41/0x60 [97443.000514] warn_alloc.cold.127+0x7b/0x108 [97443.000519] __alloc_pages_slowpath+0xcb2/0xcd0 [97443.000522] ? __alloc_pages_nodemask+0x166/0x330 [97443.000525] __alloc_pages_nodemask+0x2e2/0x330 [97443.000528] kmalloc_order+0x28/0x90 [97443.000533] kmalloc_order_trace+0x1d/0xb0 [97443.000537] __kmalloc+0x203/0x250 [97443.000541] ? __wake_up_common+0x7a/0x190 [97443.000545] virtqueue_add+0x493/0xc70 [97443.000550] ? finish_wait+0x80/0x80 [97443.000552] virtqueue_add_sgs+0x80/0xa0 [97443.000557] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97443.000562] ? scsi_alloc_sgtables+0x84/0x1c0 [97443.000566] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97443.000569] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97443.000572] scsi_queue_rq+0x512/0xb10 [97443.000574] __blk_mq_try_issue_directly+0x163/0x200 [97443.000580] blk_mq_request_issue_directly+0x4e/0xb0 [97443.000584] blk_mq_try_issue_list_directly+0x62/0x100 [97443.000587] blk_mq_sched_insert_requests+0xa4/0xf0 [97443.000591] blk_mq_flush_plug_list+0x135/0x220 [97443.000595] blk_flush_plug_list+0xd7/0x100 [97443.000599] blk_finish_plug+0x25/0x36 [97443.000604] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97443.000640] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97443.000668] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.000693] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.000718] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97443.000744] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97443.000762] ofd_commitrw+0x5b4/0xd20 [ofd] [97443.000779] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.000896] obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.000997] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97443.001115] ? newidle_balance+0x2b6/0x3b0 [97443.001119] ? internal_add_timer+0x42/0x70 [97443.001122] ? _cond_resched+0x15/0x30 [97443.001125] ? mutex_lock+0xe/0x30 [97443.001128] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97443.001225] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97443.001326] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97443.001420] ? finish_wait+0x80/0x80 [97443.001422] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97443.001525] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97443.001623] kthread+0x134/0x150 [97443.001627] ? set_kthread_struct+0x50/0x50 [97443.001629] ret_from_fork+0x1f/0x40 [97443.002680] ll_ost_io02_061: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97443.002692] CPU: 10 PID: 527336 Comm: ll_ost_io02_061 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97443.002696] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97443.002697] Call Trace: [97443.002705] ,cpuset= [97443.002701] dump_stack+0x41/0x60 [97443.002708] warn_alloc.cold.127+0x7b/0x108 [97443.002715] __alloc_pages_slowpath+0xcb2/0xcd0 [97443.002719] ? __alloc_pages_nodemask+0x166/0x330 [97443.002722] __alloc_pages_nodemask+0x2e2/0x330 [97443.002726] kmalloc_order+0x28/0x90 [97443.002731] kmalloc_order_trace+0x1d/0xb0 [97443.002735] __kmalloc+0x203/0x250 [97443.002739] ? __wake_up_common+0x7a/0x190 [97443.002743] virtqueue_add+0x493/0xc70 [97443.002748] ? finish_wait+0x80/0x80 [97443.002751] virtqueue_add_sgs+0x80/0xa0 [97443.002756] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97443.002762] ? scsi_alloc_sgtables+0x84/0x1c0 [97443.002767] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97443.002771] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97443.002775] scsi_queue_rq+0x512/0xb10 [97443.002778] __blk_mq_try_issue_directly+0x163/0x200 [97443.002783] blk_mq_request_issue_directly+0x4e/0xb0 [97443.002788] blk_mq_try_issue_list_directly+0x62/0x100 [97443.002792] blk_mq_sched_insert_requests+0xa4/0xf0 [97443.002795] blk_mq_flush_plug_list+0x135/0x220 [97443.002800] blk_flush_plug_list+0xd7/0x100 [97443.002806] blk_finish_plug+0x25/0x36 [97443.002811] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97443.002848] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97443.002880] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.002907] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.002936] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97443.002964] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97443.002988] ofd_commitrw+0x5b4/0xd20 [ofd] [97443.003007] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.003133] obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.003249] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97443.003370] ? flush_work+0x42/0x1d0 [97443.003375] ? internal_add_timer+0x42/0x70 [97443.003378] ? _cond_resched+0x15/0x30 [97443.003381] ? mutex_lock+0xe/0x30 [97443.003385] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97443.003499] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97443.003609] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97443.003727] ? finish_wait+0x80/0x80 [97443.003731] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97443.003850] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97443.003969] kthread+0x134/0x150 [97443.003973] ? set_kthread_struct+0x50/0x50 [97443.003976] ret_from_fork+0x1f/0x40 [97443.004586] / [97443.005203] ll_ost_io04_003: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [97443.005674] CPU: 17 PID: 527689 Comm: ll_ost_io04_052 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97443.006969] / [97443.006969] ,mems_allowed=0 [97443.008037] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97443.009073] ,mems_allowed=0 [97443.009933] Call Trace: [97443.010594] ll_ost_io01_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97443.011534] ll_ost_io04_020: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97443.011703] dump_stack+0x41/0x60 [97443.221653] warn_alloc.cold.127+0x7b/0x108 [97443.222595] __alloc_pages_slowpath+0xcb2/0xcd0 [97443.223597] ? __alloc_pages_nodemask+0x166/0x330 [97443.224607] __alloc_pages_nodemask+0x2e2/0x330 [97443.225592] kmalloc_order+0x28/0x90 [97443.226459] kmalloc_order_trace+0x1d/0xb0 [97443.227400] __kmalloc+0x203/0x250 [97443.228195] virtqueue_add+0x493/0xc70 [97443.229049] virtqueue_add_sgs+0x80/0xa0 [97443.229919] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97443.231044] ? scsi_alloc_sgtables+0x84/0x1c0 [97443.232016] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97443.233084] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97443.234248] scsi_queue_rq+0x512/0xb10 [97443.235091] __blk_mq_try_issue_directly+0x163/0x200 [97443.236159] blk_mq_request_issue_directly+0x4e/0xb0 [97443.237217] blk_mq_try_issue_list_directly+0x62/0x100 [97443.238347] blk_mq_sched_insert_requests+0xa4/0xf0 [97443.239384] blk_mq_flush_plug_list+0x135/0x220 [97443.240340] blk_flush_plug_list+0xd7/0x100 [97443.241265] blk_finish_plug+0x25/0x36 [97443.242106] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97443.243323] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97443.244448] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.245759] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.247064] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97443.248204] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97443.249266] ofd_commitrw+0x5b4/0xd20 [ofd] [97443.250211] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.251349] obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.252438] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97443.253558] ? flush_work+0x42/0x1d0 [97443.254371] ? internal_add_timer+0x42/0x70 [97443.255278] ? _cond_resched+0x15/0x30 [97443.256129] ? mutex_lock+0xe/0x30 [97443.256924] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97443.258082] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97443.259376] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97443.260705] ? finish_wait+0x80/0x80 [97443.261547] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97443.262610] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97443.263771] kthread+0x134/0x150 [97443.264570] ? set_kthread_struct+0x50/0x50 [97443.265484] ret_from_fork+0x1f/0x40 [97443.266288] CPU: 16 PID: 527125 Comm: ll_ost_io04_003 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97443.269268] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97443.271673] Call Trace: [97443.272282] dump_stack+0x41/0x60 [97443.273010] warn_alloc.cold.127+0x7b/0x108 [97443.273870] __alloc_pages_slowpath+0xcb2/0xcd0 [97443.274762] ? __alloc_pages_nodemask+0x166/0x330 [97443.275668] __alloc_pages_nodemask+0x2e2/0x330 [97443.276603] kmalloc_order+0x28/0x90 [97443.277377] kmalloc_order_trace+0x1d/0xb0 [97443.278163] __kmalloc+0x203/0x250 [97443.278851] ? __wake_up_common+0x7a/0x190 [97443.279635] virtqueue_add+0x493/0xc70 [97443.280349] virtqueue_add_sgs+0x80/0xa0 [97443.281112] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97443.282093] ? scsi_alloc_sgtables+0x84/0x1c0 [97443.282975] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97443.283938] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97443.284974] scsi_queue_rq+0x512/0xb10 [97443.285722] __blk_mq_try_issue_directly+0x163/0x200 [97443.286654] blk_mq_request_issue_directly+0x4e/0xb0 [97443.287585] blk_mq_try_issue_list_directly+0x62/0x100 [97443.288545] blk_mq_sched_insert_requests+0xa4/0xf0 [97443.289455] blk_mq_flush_plug_list+0x135/0x220 [97443.290335] blk_flush_plug_list+0xd7/0x100 [97443.291158] blk_finish_plug+0x25/0x36 [97443.291904] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97443.292992] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97443.293997] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.295183] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.296317] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97443.297326] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97443.298237] ofd_commitrw+0x5b4/0xd20 [ofd] [97443.299034] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.299988] obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.300925] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97443.301893] ? flush_work+0x42/0x1d0 [97443.302614] ? internal_add_timer+0x42/0x70 [97443.303424] ? _cond_resched+0x15/0x30 [97443.304174] ? mutex_lock+0xe/0x30 [97443.304866] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97443.305878] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97443.306962] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97443.308107] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97443.309058] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97443.310068] kthread+0x134/0x150 [97443.310722] ? set_kthread_struct+0x50/0x50 [97443.311498] ret_from_fork+0x1f/0x40 [97443.312187] CPU: 9 PID: 527212 Comm: ll_ost_io02_036 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97443.314872] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97443.316460] Call Trace: [97443.317104] dump_stack+0x41/0x60 [97443.317801] warn_alloc.cold.127+0x7b/0x108 [97443.318727] __alloc_pages_slowpath+0xcb2/0xcd0 [97443.319658] ? __alloc_pages_nodemask+0x166/0x330 [97443.320564] __alloc_pages_nodemask+0x2e2/0x330 [97443.321566] kmalloc_order+0x28/0x90 [97443.322373] kmalloc_order_trace+0x1d/0xb0 [97443.323405] __kmalloc+0x203/0x250 [97443.324179] ? __wake_up_common+0x7a/0x190 [97443.324994] virtqueue_add+0x493/0xc70 [97443.325941] ? finish_wait+0x80/0x80 [97443.326889] virtqueue_add_sgs+0x80/0xa0 [97443.327876] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97443.329070] ? scsi_alloc_sgtables+0x84/0x1c0 [97443.330171] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97443.331394] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97443.332744] scsi_queue_rq+0x512/0xb10 [97443.333725] __blk_mq_try_issue_directly+0x163/0x200 [97443.334947] blk_mq_request_issue_directly+0x4e/0xb0 [97443.336175] blk_mq_try_issue_list_directly+0x62/0x100 [97443.337425] blk_mq_sched_insert_requests+0xa4/0xf0 [97443.338625] blk_mq_flush_plug_list+0x135/0x220 [97443.339754] blk_flush_plug_list+0xd7/0x100 [97443.340828] blk_finish_plug+0x25/0x36 [97443.341812] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97443.343005] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97443.344182] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.345695] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.347164] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97443.348472] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97443.349674] ofd_commitrw+0x5b4/0xd20 [ofd] [97443.350749] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.352042] obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.353355] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97443.354622] ? flush_work+0x42/0x1d0 [97443.355405] ? internal_add_timer+0x42/0x70 [97443.356186] ? _cond_resched+0x15/0x30 [97443.356906] ? mutex_lock+0xe/0x30 [97443.357612] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97443.358981] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97443.360511] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97443.362061] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97443.363323] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97443.364737] kthread+0x134/0x150 [97443.365489] ? set_kthread_struct+0x50/0x50 [97443.366500] ret_from_fork+0x1f/0x40 [97443.367293] CPU: 18 PID: 527235 Comm: ll_ost_io04_020 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97443.369620] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97443.371187] Call Trace: [97443.371782] dump_stack+0x41/0x60 [97443.372496] warn_alloc.cold.127+0x7b/0x108 [97443.373341] __alloc_pages_slowpath+0xcb2/0xcd0 [97443.374260] ? __alloc_pages_nodemask+0x166/0x330 [97443.375152] __alloc_pages_nodemask+0x2e2/0x330 [97443.376012] kmalloc_order+0x28/0x90 [97443.376738] kmalloc_order_trace+0x1d/0xb0 [97443.377588] __kmalloc+0x203/0x250 [97443.378315] ? __wake_up_common+0x7a/0x190 [97443.379134] virtqueue_add+0x493/0xc70 [97443.379917] ? finish_wait+0x80/0x80 [97443.380659] virtqueue_add_sgs+0x80/0xa0 [97443.381483] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97443.382567] ? scsi_alloc_sgtables+0x84/0x1c0 [97443.383431] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97443.384365] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97443.385405] scsi_queue_rq+0x512/0xb10 [97443.386159] __blk_mq_try_issue_directly+0x163/0x200 [97443.387096] blk_mq_request_issue_directly+0x4e/0xb0 [97443.388031] blk_mq_try_issue_list_directly+0x62/0x100 [97443.388971] blk_mq_sched_insert_requests+0xa4/0xf0 [97443.389876] blk_mq_flush_plug_list+0x135/0x220 [97443.391026] blk_flush_plug_list+0xd7/0x100 [97443.392108] blk_finish_plug+0x25/0x36 [97443.393029] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97443.394084] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97443.395047] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.396162] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.397259] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97443.398234] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97443.399134] ofd_commitrw+0x5b4/0xd20 [ofd] [97443.399934] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.400873] obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.401797] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97443.402753] ? kvm_set_wallclock+0x10/0x10 [97443.403579] ? sched_clock+0x5/0x10 [97443.404293] ? newidle_balance+0x1e2/0x3b0 [97443.405067] ? internal_add_timer+0x42/0x70 [97443.405867] ? _cond_resched+0x15/0x30 [97443.406593] ? mutex_lock+0xe/0x30 [97443.407287] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97443.408281] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97443.409384] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97443.410465] ? finish_wait+0x80/0x80 [97443.411165] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97443.412082] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97443.413072] kthread+0x134/0x150 [97443.413727] ? set_kthread_struct+0x50/0x50 [97443.414503] ret_from_fork+0x1f/0x40 [97443.415196] CPU: 7 PID: 527199 Comm: ll_ost_io01_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97443.418191] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97443.420306] Call Trace: [97443.421073] dump_stack+0x41/0x60 [97443.421994] warn_alloc.cold.127+0x7b/0x108 [97443.423105] __alloc_pages_slowpath+0xcb2/0xcd0 [97443.424271] ? __alloc_pages_nodemask+0x166/0x330 [97443.425467] __alloc_pages_nodemask+0x2e2/0x330 [97443.426634] kmalloc_order+0x28/0x90 [97443.427635] kmalloc_order_trace+0x1d/0xb0 [97443.428729] __kmalloc+0x203/0x250 [97443.429688] ? __wake_up_common+0x7a/0x190 [97443.430811] virtqueue_add+0x493/0xc70 [97443.431850] ? finish_wait+0x80/0x80 [97443.432840] virtqueue_add_sgs+0x80/0xa0 [97443.433925] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97443.435290] ? scsi_alloc_sgtables+0x84/0x1c0 [97443.436431] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97443.437703] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97443.439087] scsi_queue_rq+0x512/0xb10 [97443.440111] __blk_mq_try_issue_directly+0x163/0x200 [97443.441367] blk_mq_request_issue_directly+0x4e/0xb0 [97443.442656] blk_mq_try_issue_list_directly+0x62/0x100 [97443.443951] blk_mq_sched_insert_requests+0xa4/0xf0 [97443.445210] blk_mq_flush_plug_list+0x135/0x220 [97443.446414] blk_flush_plug_list+0xd7/0x100 [97443.447510] blk_finish_plug+0x25/0x36 [97443.448512] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97443.449971] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97443.451335] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.452918] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97443.454445] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97443.455806] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97443.457088] ofd_commitrw+0x5b4/0xd20 [ofd] [97443.458227] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.459604] obd_commitrw+0x1b6/0x370 [ptlrpc] [97443.460935] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97443.462318] ? newidle_balance+0x2b6/0x3b0 [97443.463430] ? internal_add_timer+0x42/0x70 [97443.464556] ? _cond_resched+0x15/0x30 [97443.465574] ? mutex_lock+0xe/0x30 [97443.466553] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97443.467974] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97443.469536] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97443.471122] ? finish_wait+0x80/0x80 [97443.472132] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97443.473473] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97443.474921] kthread+0x134/0x150 [97443.475847] ? set_kthread_struct+0x50/0x50 [97443.476973] ret_from_fork+0x1f/0x40 [97447.054525] ll_ost_io04_053: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [97447.057473] CPU: 17 PID: 527690 Comm: ll_ost_io04_053 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97447.058179] ll_ost_io02_009: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [97447.060460] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97447.060461] ,cpuset=/ [97447.062839] Call Trace: [97447.064979] ,mems_allowed=0 [97447.065790] dump_stack+0x41/0x60 [97447.067472] warn_alloc.cold.127+0x7b/0x108 [97447.070256] __alloc_pages_slowpath+0xcb2/0xcd0 [97447.071445] ? __alloc_pages_nodemask+0x166/0x330 [97447.072644] __alloc_pages_nodemask+0x2e2/0x330 [97447.073824] kmalloc_order+0x28/0x90 [97447.074807] kmalloc_order_trace+0x1d/0xb0 [97447.075881] __kmalloc+0x203/0x250 [97447.076834] virtqueue_add+0x493/0xc70 [97447.077849] virtqueue_add_sgs+0x80/0xa0 [97447.078912] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97447.080296] ? scsi_alloc_sgtables+0x84/0x1c0 [97447.081434] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97447.082691] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97447.084056] scsi_queue_rq+0x512/0xb10 [97447.085047] __blk_mq_try_issue_directly+0x163/0x200 [97447.086288] blk_mq_request_issue_directly+0x4e/0xb0 [97447.087542] blk_mq_try_issue_list_directly+0x62/0x100 [97447.088757] blk_mq_sched_insert_requests+0xa4/0xf0 [97447.089809] blk_mq_flush_plug_list+0x135/0x220 [97447.090896] blk_flush_plug_list+0xd7/0x100 [97447.091971] blk_finish_plug+0x25/0x36 [97447.092957] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97447.094047] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97447.095185] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97447.096521] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97447.098036] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97447.099359] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97447.100573] ofd_commitrw+0x5b4/0xd20 [ofd] [97447.101675] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97447.102977] obd_commitrw+0x1b6/0x370 [ptlrpc] [97447.104252] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97447.105564] ? newidle_balance+0x2b6/0x3b0 [97447.106625] ? internal_add_timer+0x42/0x70 [97447.107726] ? _cond_resched+0x15/0x30 [97447.108724] ? mutex_lock+0xe/0x30 [97447.109654] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97447.111025] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97447.112545] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97447.114076] ? finish_wait+0x80/0x80 [97447.115071] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97447.116368] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97447.117746] kthread+0x134/0x150 [97447.118634] ? set_kthread_struct+0x50/0x50 [97447.119681] ret_from_fork+0x1f/0x40 [97447.120613] warn_alloc_show_mem: 7 callbacks suppressed [97447.120616] Mem-Info: [97447.120614] CPU: 9 PID: 527052 Comm: ll_ost_io02_009 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [97447.121854] active_anon:27878 inactive_anon:199834 isolated_anon:0 active_file:30730989 inactive_file:3451290 isolated_file:0 unevictable:0 dirty:1281 writeback:0 slab_reclaimable:1000509 slab_unreclaimable:604888 mapped:23720 shmem:146 pagetables:5708 bounce:0 free:1996286 free_pcp:388 free_cma:0 [97447.122623] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [97447.125335] Node 0 active_anon:111512kB inactive_anon:799336kB active_file:122928072kB inactive_file:13798440kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:94880kB dirty:5124kB writeback:0kB shmem:584kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35504kB pagetables:22832kB all_unreclaimable? no [97447.132436] Call Trace: [97447.134590] Node 0 [97447.140063] dump_stack+0x41/0x60 [97447.140880] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97447.141585] warn_alloc.cold.127+0x7b/0x108 [97447.142555] lowmem_reserve[]: [97447.147365] __alloc_pages_slowpath+0xcb2/0xcd0 [97447.148520] 0 [97447.149308] ? __alloc_pages_nodemask+0x166/0x330 [97447.150473] 913 [97447.151130] __alloc_pages_nodemask+0x2e2/0x330 [97447.152365] 150007 [97447.152908] kmalloc_order+0x28/0x90 [97447.154142] 150007 [97447.154890] kmalloc_order_trace+0x1d/0xb0 [97447.155930] 150007 [97447.156703] __kmalloc+0x203/0x250 [97447.158656] ? __wake_up_common+0x7a/0x190 [97447.159679] Node 0 [97447.160308] virtqueue_add+0x493/0xc70 [97447.161502] DMA32 free:596768kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [97447.162057] virtqueue_add_sgs+0x80/0xa0 [97447.163126] lowmem_reserve[]: [97447.167165] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [97447.168329] 0 [97447.169069] ? scsi_alloc_sgtables+0x84/0x1c0 [97447.170522] 0 [97447.171054] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [97447.172228] 149093 [97447.172755] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [97447.173853] 149093 [97447.174479] scsi_queue_rq+0x512/0xb10 [97447.175784] 149093 [97447.176357] __blk_mq_try_issue_directly+0x163/0x200 [97447.178024] blk_mq_request_issue_directly+0x4e/0xb0 [97447.179411] Node 0 [97447.179900] blk_mq_try_issue_list_directly+0x62/0x100 [97447.181237] Normal free:7841472kB min:49268kB low:201940kB high:354612kB active_anon:111132kB inactive_anon:799332kB active_file:122486080kB inactive_file:13778376kB unevictable:0kB writepending:5124kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2020kB local_pcp:16kB free_cma:0kB [97447.181799] blk_mq_sched_insert_requests+0xa4/0xf0 [97447.183204] lowmem_reserve[]: [97447.187446] blk_mq_flush_plug_list+0x135/0x220 [97447.188778] 0 [97447.189452] blk_flush_plug_list+0xd7/0x100 [97447.190680] 0 [97447.191174] blk_finish_plug+0x25/0x36 [97447.192364] 0 [97447.193060] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [97447.194146] 0 [97447.194704] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [97447.196185] 0 [97447.196772] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97447.198803] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [97447.200384] Node 0 [97447.201035] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [97447.202535] DMA: [97447.203233] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [97447.204566] 0*4kB [97447.205279] ofd_commitrw+0x5b4/0xd20 [ofd] [97447.206527] 0*8kB [97447.207237] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [97447.208384] 0*16kB [97447.209104] obd_commitrw+0x1b6/0x370 [ptlrpc] [97447.210303] 0*32kB [97447.211037] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [97447.212195] 0*64kB [97447.212943] ? flush_work+0x42/0x1d0 [97447.214195] 0*128kB [97447.214809] ? internal_add_timer+0x42/0x70 [97447.215853] 0*256kB [97447.216589] ? _cond_resched+0x15/0x30 [97447.217755] 0*512kB [97447.218367] ? mutex_lock+0xe/0x30 [97447.219410] 1*1024kB [97447.220005] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [97447.220971] (U) [97447.221555] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [97447.222824] 1*2048kB [97447.223499] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [97447.224913] (M) [97447.225590] ptlrpc_main+0xbec/0x1530 [ptlrpc] [97447.227062] 2*4096kB [97447.227776] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [97447.228939] (M) [97447.229669] kthread+0x134/0x150 [97447.230976] = 11264kB [97447.231677] ? set_kthread_struct+0x50/0x50 [97447.232658] Node 0 [97447.233417] ret_from_fork+0x1f/0x40 [97447.234541] DMA32: 18*4kB (UM) 21*8kB (UM) 39*16kB (UME) 54*32kB (UMH) 47*64kB (UMH) 45*128kB (UM) 46*256kB (UMH) 45*512kB (UME) 38*1024kB (UM) 42*2048kB (UM) 104*4096kB (UM) = 597088kB [97447.240074] Node 0 Normal: 1168576*4kB (UMEH) 363558*8kB (UMEH) 16900*16kB (UMEH) 15*32kB (UH) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 7853648kB [97447.243362] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [97447.245355] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [97447.247261] 34043251 total pagecache pages [97447.248370] 1651 pages in swap cache [97447.249353] Swap cache stats: add 1612211, delete 1610583, find 84352/101786 [97447.250984] Free swap = 5322268kB [97447.251937] Total swap = 11075580kB [97447.252892] 39321462 pages RAM [97447.253759] 0 pages HighMem/MovableOnly [97447.254798] 905848 pages reserved [97447.255713] 0 pages hwpoisoned [99824.381079] warn_alloc: 13 callbacks suppressed [99824.381083] ll_ost_io02_042: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.384715] CPU: 8 PID: 527247 Comm: ll_ost_io02_042 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.386875] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.387245] ll_ost_io01_027: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.388685] Call Trace: [99824.392287] dump_stack+0x41/0x60 [99824.393261] warn_alloc.cold.127+0x7b/0x108 [99824.394283] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.395358] ? __alloc_pages_nodemask+0x166/0x330 [99824.396584] __alloc_pages_nodemask+0x2e2/0x330 [99824.397557] kmalloc_order+0x28/0x90 [99824.398335] kmalloc_order_trace+0x1d/0xb0 [99824.399261] __kmalloc+0x203/0x250 [99824.400196] virtqueue_add+0x493/0xc70 [99824.401023] virtqueue_add_sgs+0x80/0xa0 [99824.402049] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.403407] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.404412] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.405426] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.406499] scsi_queue_rq+0x512/0xb10 [99824.407336] __blk_mq_try_issue_directly+0x163/0x200 [99824.408337] blk_mq_request_issue_directly+0x4e/0xb0 [99824.409340] blk_mq_try_issue_list_directly+0x62/0x100 [99824.410324] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.411242] blk_mq_flush_plug_list+0x135/0x220 [99824.412135] blk_flush_plug_list+0xd7/0x100 [99824.412966] blk_finish_plug+0x25/0x36 [99824.413729] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.414874] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.415999] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.417112] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.418093] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.419067] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.419922] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.420902] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.421840] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.422783] ? flush_work+0x42/0x1d0 [99824.423473] ? internal_add_timer+0x42/0x70 [99824.424275] ? _cond_resched+0x15/0x30 [99824.425025] ? mutex_lock+0xe/0x30 [99824.425721] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.426721] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.427796] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.428883] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.429770] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.430765] kthread+0x134/0x150 [99824.431434] ? set_kthread_struct+0x50/0x50 [99824.432209] ret_from_fork+0x1f/0x40 [99824.432900] warn_alloc_show_mem: 1 callbacks suppressed [99824.432902] Mem-Info: [99824.432901] CPU: 5 PID: 527110 Comm: ll_ost_io01_027 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.433831] active_anon:27888 inactive_anon:171261 isolated_anon:0 active_file:27781924 inactive_file:7234601 isolated_file:0 unevictable:0 dirty:1458 writeback:0 slab_reclaimable:1010515 slab_unreclaimable:605588 mapped:20790 shmem:145 pagetables:5783 bounce:0 free:1181610 free_pcp:583 free_cma:0 [99824.434544] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.436600] Node 0 active_anon:111552kB inactive_anon:685044kB active_file:111101560kB inactive_file:28934308kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:83160kB dirty:5832kB writeback:0kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35504kB pagetables:23132kB all_unreclaimable? no [99824.444376] Call Trace: [99824.446527] Node 0 [99824.453485] dump_stack+0x41/0x60 [99824.454313] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [99824.455071] warn_alloc.cold.127+0x7b/0x108 [99824.456108] lowmem_reserve[]: [99824.460843] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.462021] 0 [99824.463005] ? __alloc_pages_nodemask+0x166/0x330 [99824.464202] 913 [99824.464890] __alloc_pages_nodemask+0x2e2/0x330 [99824.466090] 150007 [99824.466795] kmalloc_order+0x28/0x90 [99824.467765] 150007 [99824.468553] kmalloc_order_trace+0x1d/0xb0 [99824.469338] 150007 [99824.470097] __kmalloc+0x203/0x250 [99824.471761] virtqueue_add+0x493/0xc70 [99824.472513] Node 0 [99824.473162] virtqueue_add_sgs+0x80/0xa0 [99824.473959] DMA32 free:596688kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:60kB inactive_file:16kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:4kB local_pcp:0kB free_cma:0kB [99824.474741] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.475846] lowmem_reserve[]: [99824.481263] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.482721] 0 [99824.483718] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.484965] 0 [99824.485673] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.487045] 149093 [99824.487763] scsi_queue_rq+0x512/0xb10 [99824.489262] 149093 [99824.490072] __blk_mq_try_issue_directly+0x163/0x200 [99824.491194] 149093 [99824.491995] blk_mq_request_issue_directly+0x4e/0xb0 [99824.494162] blk_mq_try_issue_list_directly+0x62/0x100 [99824.495534] Node 0 [99824.496243] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.497636] Normal free:4184664kB min:49268kB low:201940kB high:354612kB active_anon:111172kB inactive_anon:685040kB active_file:111058548kB inactive_file:28943532kB unevictable:0kB writepending:6636kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1588kB local_pcp:24kB free_cma:0kB [99824.498430] blk_mq_flush_plug_list+0x135/0x220 [99824.499718] lowmem_reserve[]: 0 [99824.504458] blk_flush_plug_list+0xd7/0x100 [99824.505700] 0 [99824.506496] blk_finish_plug+0x25/0x36 [99824.507666] 0 [99824.508254] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.509241] 0 [99824.509802] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [99824.511142] 0 [99824.511746] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.513648] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.515238] Node 0 [99824.515869] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.517456] DMA: [99824.518098] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.519114] 0*4kB [99824.519687] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.520639] 0*8kB [99824.521232] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.522090] 0*16kB [99824.522675] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.523589] 0*32kB [99824.524167] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.525032] 0*64kB [99824.525646] ? flush_work+0x42/0x1d0 [99824.526578] 0*128kB [99824.527179] ? internal_add_timer+0x42/0x70 [99824.527983] 0*256kB [99824.528587] ? _cond_resched+0x15/0x30 [99824.529445] 0*512kB [99824.530092] ? mutex_lock+0xe/0x30 [99824.530945] 1*1024kB [99824.531529] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.532497] (U) [99824.533128] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.534160] 1*2048kB [99824.534813] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.536128] (M) [99824.536806] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.538210] 2*4096kB [99824.538885] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.539948] (M) [99824.540588] kthread+0x134/0x150 [99824.541923] = 11264kB [99824.542476] ? set_kthread_struct+0x50/0x50 [99824.543415] Node 0 [99824.544039] ret_from_fork+0x1f/0x40 [99824.545239] DMA32: 22*4kB (UMH) 33*8kB (UMH) 29*16kB (UMEH) 22*32kB (UME) 32*64kB (UMH) 44*128kB (UM) 44*256kB (UM) 45*512kB (UMEH) 40*1024kB (UMH) 42*2048kB (UM) 104*4096kB (UM) = 596464kB [99824.551545] Node 0 Normal: 842121*4kB (UME) 92800*8kB (UME) 5592*16kB (UMH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 4200356kB [99824.554765] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [99824.556195] ll_ost_io01_021: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [99824.557045] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [99824.557048] 34995240 total pagecache pages [99824.557352] ll_ost_io03_075: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.557363] CPU: 14 PID: 527460 Comm: ll_ost_io03_075 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.557367] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.557368] Call Trace: [99824.557372] dump_stack+0x41/0x60 [99824.557380] warn_alloc.cold.127+0x7b/0x108 [99824.557387] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.557391] ? __alloc_pages_nodemask+0x166/0x330 [99824.557394] __alloc_pages_nodemask+0x2e2/0x330 [99824.557397] kmalloc_order+0x28/0x90 [99824.557403] kmalloc_order_trace+0x1d/0xb0 [99824.557406] __kmalloc+0x203/0x250 [99824.557412] virtqueue_add+0x493/0xc70 [99824.557418] ? finish_wait+0x80/0x80 [99824.557423] virtqueue_add_sgs+0x80/0xa0 [99824.557428] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.557434] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.557440] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.557443] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.557447] scsi_queue_rq+0x512/0xb10 [99824.557451] __blk_mq_try_issue_directly+0x163/0x200 [99824.557457] blk_mq_request_issue_directly+0x4e/0xb0 [99824.557462] blk_mq_try_issue_list_directly+0x62/0x100 [99824.557466] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.557470] blk_mq_flush_plug_list+0x135/0x220 [99824.557474] blk_flush_plug_list+0xd7/0x100 [99824.557480] blk_finish_plug+0x25/0x36 [99824.557485] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.557524] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [99824.557557] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.557587] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.557615] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.557645] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.557669] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.557686] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.557811] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.557926] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.558039] ? flush_work+0x42/0x1d0 [99824.558044] ? internal_add_timer+0x42/0x70 [99824.558048] ? _cond_resched+0x15/0x30 [99824.558051] ? mutex_lock+0xe/0x30 [99824.558054] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.558164] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.558272] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.558359] ll_ost_io04_070: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.558386] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.558495] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.558599] kthread+0x134/0x150 [99824.558603] ? set_kthread_struct+0x50/0x50 [99824.558607] ret_from_fork+0x1f/0x40 [99824.558613] CPU: 19 PID: 528058 Comm: ll_ost_io04_070 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.558617] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.558619] Call Trace: [99824.558622] dump_stack+0x41/0x60 [99824.558629] warn_alloc.cold.127+0x7b/0x108 [99824.558635] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.558640] __alloc_pages_nodemask+0x2e2/0x330 [99824.558643] kmalloc_order+0x28/0x90 [99824.558649] kmalloc_order_trace+0x1d/0xb0 [99824.558653] __kmalloc+0x203/0x250 [99824.558658] virtqueue_add+0x493/0xc70 [99824.558663] ? finish_wait+0x80/0x80 [99824.558668] virtqueue_add_sgs+0x80/0xa0 [99824.558673] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.558679] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.558684] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.558688] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.558692] scsi_queue_rq+0x512/0xb10 [99824.558695] __blk_mq_try_issue_directly+0x163/0x200 [99824.558701] blk_mq_request_issue_directly+0x4e/0xb0 [99824.558705] blk_mq_try_issue_list_directly+0x62/0x100 [99824.558709] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.558713] blk_mq_flush_plug_list+0x135/0x220 [99824.558718] blk_flush_plug_list+0xd7/0x100 [99824.558723] blk_finish_plug+0x25/0x36 [99824.558728] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.558764] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [99824.558794] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.558820] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.558849] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.558877] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.558898] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.558917] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.559039] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.559132] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.559205] ? flush_work+0x42/0x1d0 [99824.559209] ? internal_add_timer+0x42/0x70 [99824.559211] ? _cond_resched+0x15/0x30 [99824.559213] ? mutex_lock+0xe/0x30 [99824.559215] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.559294] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.559367] / [99824.559387] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.559457] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.559555] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.559622] kthread+0x134/0x150 [99824.559625] ? set_kthread_struct+0x50/0x50 [99824.559628] ret_from_fork+0x1f/0x40 [99824.561535] 1305 pages in swap cache [99824.562424] ,mems_allowed=0 [99824.565360] Swap cache stats: add 1704918, delete 1703637, find 96270/116425 [99824.570514] Free swap = 5051252kB [99824.571264] CPU: 6 PID: 527078 Comm: ll_ost_io01_021 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.571343] ll_ost_io04_018: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.572067] Total swap = 11075580kB [99824.573049] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.574009] 39321462 pages RAM [99824.574930] Call Trace: [99824.575966] 0 pages HighMem/MovableOnly [99824.576804] dump_stack+0x41/0x60 [99824.577803] 905848 pages reserved [99824.578502] ll_ost_io04_057: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.578613] warn_alloc.cold.127+0x7b/0x108 [99824.579415] 0 pages hwpoisoned [99824.580172] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.649665] ll_ost_io00_064: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [99824.649680] ll_ost_io00_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.649780] ll_ost_io00_052: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [99824.650322] ? __alloc_pages_nodemask+0x166/0x330 [99824.651730] ,cpuset= [99824.652854] __alloc_pages_nodemask+0x2e2/0x330 [99824.654033] / [99824.654939] kmalloc_order+0x28/0x90 [99824.655920] ,mems_allowed=0 [99824.656819] kmalloc_order_trace+0x1d/0xb0 [99824.658767] __kmalloc+0x203/0x250 [99824.697014] virtqueue_add+0x493/0xc70 [99824.697742] virtqueue_add_sgs+0x80/0xa0 [99824.698490] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.699440] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.700268] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.701405] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.702753] scsi_queue_rq+0x512/0xb10 [99824.703553] __blk_mq_try_issue_directly+0x163/0x200 [99824.704531] blk_mq_request_issue_directly+0x4e/0xb0 [99824.705481] blk_mq_try_issue_list_directly+0x62/0x100 [99824.706538] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.707588] blk_mq_flush_plug_list+0x135/0x220 [99824.708741] blk_flush_plug_list+0xd7/0x100 [99824.709663] blk_finish_plug+0x25/0x36 [99824.710478] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.711676] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.713220] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.714560] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.715614] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.716777] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.717865] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.719169] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.720483] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.721834] ? newidle_balance+0x2b6/0x3b0 [99824.722921] ? internal_add_timer+0x42/0x70 [99824.724010] ? _cond_resched+0x15/0x30 [99824.725000] ? mutex_lock+0xe/0x30 [99824.725915] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.727082] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.728213] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.729592] ? finish_wait+0x80/0x80 [99824.730494] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.731433] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.732452] kthread+0x134/0x150 [99824.733119] ? set_kthread_struct+0x50/0x50 [99824.734000] ret_from_fork+0x1f/0x40 [99824.734830] CPU: 3 PID: 527429 Comm: ll_ost_io00_052 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.737569] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.739166] Call Trace: [99824.739714] dump_stack+0x41/0x60 [99824.740406] warn_alloc.cold.127+0x7b/0x108 [99824.741185] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.742033] ? __alloc_pages_nodemask+0x166/0x330 [99824.742894] __alloc_pages_nodemask+0x2e2/0x330 [99824.743738] kmalloc_order+0x28/0x90 [99824.744489] kmalloc_order_trace+0x1d/0xb0 [99824.745389] __kmalloc+0x203/0x250 [99824.746172] virtqueue_add+0x493/0xc70 [99824.746887] ? finish_wait+0x80/0x80 [99824.747581] virtqueue_add_sgs+0x80/0xa0 [99824.748325] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.749272] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.750081] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.750985] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.751984] scsi_queue_rq+0x512/0xb10 [99824.752692] __blk_mq_try_issue_directly+0x163/0x200 [99824.753565] blk_mq_request_issue_directly+0x4e/0xb0 [99824.754436] blk_mq_try_issue_list_directly+0x62/0x100 [99824.755336] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.756439] blk_mq_flush_plug_list+0x135/0x220 [99824.757265] blk_flush_plug_list+0xd7/0x100 [99824.758040] blk_finish_plug+0x25/0x36 [99824.758750] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.759794] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [99824.760747] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.761869] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.762958] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.763917] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.764801] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.765563] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.766482] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.767377] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.768301] ? flush_work+0x42/0x1d0 [99824.769064] ? internal_add_timer+0x42/0x70 [99824.770180] ? _cond_resched+0x15/0x30 [99824.771039] ? mutex_lock+0xe/0x30 [99824.771705] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.772732] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.773825] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.774901] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.775779] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.776740] kthread+0x134/0x150 [99824.777375] ? set_kthread_struct+0x50/0x50 [99824.778278] ret_from_fork+0x1f/0x40 [99824.778964] CPU: 18 PID: 527233 Comm: ll_ost_io04_018 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.781323] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.783014] Call Trace: [99824.783642] dump_stack+0x41/0x60 [99824.784407] warn_alloc.cold.127+0x7b/0x108 [99824.785312] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.786276] __alloc_pages_nodemask+0x2e2/0x330 [99824.787227] kmalloc_order+0x28/0x90 [99824.788027] kmalloc_order_trace+0x1d/0xb0 [99824.788900] __kmalloc+0x203/0x250 [99824.789661] virtqueue_add+0x493/0xc70 [99824.790494] ? finish_wait+0x80/0x80 [99824.791302] virtqueue_add_sgs+0x80/0xa0 [99824.792155] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.793242] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.794178] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.795210] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.796350] scsi_queue_rq+0x512/0xb10 [99824.797186] __blk_mq_try_issue_directly+0x163/0x200 [99824.798193] blk_mq_request_issue_directly+0x4e/0xb0 [99824.799218] blk_mq_try_issue_list_directly+0x62/0x100 [99824.800263] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.801260] blk_mq_flush_plug_list+0x135/0x220 [99824.802192] blk_flush_plug_list+0xd7/0x100 [99824.803101] blk_finish_plug+0x25/0x36 [99824.803952] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.805141] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [99824.806244] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.807502] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.808723] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.809817] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.810833] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.811723] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.812787] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.813822] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.814889] ? __mutex_lock_slowpath+0x10/0x10 [99824.815829] ? _cond_resched+0x15/0x30 [99824.816665] ? mutex_lock+0xe/0x30 [99824.817444] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.818600] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.819860] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.821140] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.822176] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.823299] kthread+0x134/0x150 [99824.824057] ? set_kthread_struct+0x50/0x50 [99824.824959] ret_from_fork+0x1f/0x40 [99824.825778] CPU: 1 PID: 527757 Comm: ll_ost_io00_064 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.828304] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.830120] Call Trace: [99824.830779] dump_stack+0x41/0x60 [99824.831568] warn_alloc.cold.127+0x7b/0x108 [99824.832516] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.833506] __alloc_pages_nodemask+0x2e2/0x330 [99824.834517] kmalloc_order+0x28/0x90 [99824.835368] kmalloc_order_trace+0x1d/0xb0 [99824.836307] __kmalloc+0x203/0x250 [99824.837130] virtqueue_add+0x493/0xc70 [99824.837989] ? finish_wait+0x80/0x80 [99824.838813] virtqueue_add_sgs+0x80/0xa0 [99824.839713] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.840855] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.841821] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.842902] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.844093] scsi_queue_rq+0x512/0xb10 [99824.844950] __blk_mq_try_issue_directly+0x163/0x200 [99824.846017] blk_mq_request_issue_directly+0x4e/0xb0 [99824.847081] blk_mq_try_issue_list_directly+0x62/0x100 [99824.848184] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.849235] blk_mq_flush_plug_list+0x135/0x220 [99824.850222] blk_flush_plug_list+0xd7/0x100 [99824.851156] blk_finish_plug+0x25/0x36 [99824.852015] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.853263] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [99824.854405] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.855727] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.857025] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.858161] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.859230] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.860213] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.861368] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.862478] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.863604] ? newidle_balance+0x2b6/0x3b0 [99824.864529] ? internal_add_timer+0x42/0x70 [99824.865462] ? _cond_resched+0x15/0x30 [99824.866316] ? mutex_lock+0xe/0x30 [99824.867110] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.868298] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.869585] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.870895] ? finish_wait+0x80/0x80 [99824.871757] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.872888] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.874096] kthread+0x134/0x150 [99824.874865] ? set_kthread_struct+0x50/0x50 [99824.875791] ret_from_fork+0x1f/0x40 [99824.876625] CPU: 19 PID: 527694 Comm: ll_ost_io04_057 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.879486] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.881569] Call Trace: [99824.882331] dump_stack+0x41/0x60 [99824.883255] warn_alloc.cold.127+0x7b/0x108 [99824.884334] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.885481] ? __alloc_pages_nodemask+0x166/0x330 [99824.886709] __alloc_pages_nodemask+0x2e2/0x330 [99824.887892] kmalloc_order+0x28/0x90 [99824.888852] kmalloc_order_trace+0x1d/0xb0 [99824.889929] __kmalloc+0x203/0x250 [99824.890897] virtqueue_add+0x493/0xc70 [99824.891931] virtqueue_add_sgs+0x80/0xa0 [99824.892979] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.894351] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.895502] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.896755] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.898208] scsi_queue_rq+0x512/0xb10 [99824.899255] __blk_mq_try_issue_directly+0x163/0x200 [99824.900516] blk_mq_request_issue_directly+0x4e/0xb0 [99824.901752] blk_mq_try_issue_list_directly+0x62/0x100 [99824.903036] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.904221] blk_mq_flush_plug_list+0x135/0x220 [99824.905358] blk_flush_plug_list+0xd7/0x100 [99824.906468] blk_finish_plug+0x25/0x36 [99824.907482] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.908924] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.910459] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.912007] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.913348] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.914607] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.915738] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.917051] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.918323] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.919671] ? flush_work+0x42/0x1d0 [99824.920682] ? internal_add_timer+0x42/0x70 [99824.921778] ? _cond_resched+0x15/0x30 [99824.922774] ? mutex_lock+0xe/0x30 [99824.923704] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.925081] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.926607] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.928140] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.929440] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.930829] kthread+0x134/0x150 [99824.931744] ? set_kthread_struct+0x50/0x50 [99824.932827] ret_from_fork+0x1f/0x40 [99824.933810] CPU: 0 PID: 527129 Comm: ll_ost_io00_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [99824.937328] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [99824.939710] Call Trace: [99824.940669] dump_stack+0x41/0x60 [99824.941689] warn_alloc.cold.127+0x7b/0x108 [99824.942756] __alloc_pages_slowpath+0xcb2/0xcd0 [99824.943630] __alloc_pages_nodemask+0x2e2/0x330 [99824.944620] kmalloc_order+0x28/0x90 [99824.945465] kmalloc_order_trace+0x1d/0xb0 [99824.946381] __kmalloc+0x203/0x250 [99824.947055] virtqueue_add+0x493/0xc70 [99824.947861] ? finish_wait+0x80/0x80 [99824.948619] virtqueue_add_sgs+0x80/0xa0 [99824.949635] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [99824.951077] ? scsi_alloc_sgtables+0x84/0x1c0 [99824.952263] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [99824.953581] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [99824.954848] scsi_queue_rq+0x512/0xb10 [99824.955732] __blk_mq_try_issue_directly+0x163/0x200 [99824.956774] blk_mq_request_issue_directly+0x4e/0xb0 [99824.957845] blk_mq_try_issue_list_directly+0x62/0x100 [99824.958850] blk_mq_sched_insert_requests+0xa4/0xf0 [99824.959786] blk_mq_flush_plug_list+0x135/0x220 [99824.960735] blk_flush_plug_list+0xd7/0x100 [99824.961565] blk_finish_plug+0x25/0x36 [99824.962311] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [99824.963575] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [99824.964992] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.966381] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [99824.967894] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [99824.969371] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [99824.970721] ofd_commitrw+0x5b4/0xd20 [ofd] [99824.971897] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.973302] obd_commitrw+0x1b6/0x370 [ptlrpc] [99824.974678] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [99824.976074] ? newidle_balance+0x2b6/0x3b0 [99824.977212] ? internal_add_timer+0x42/0x70 [99824.978378] ? _cond_resched+0x15/0x30 [99824.979840] ? mutex_lock+0xe/0x30 [99824.980880] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [99824.982370] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [99824.983995] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [99824.985608] ? finish_wait+0x80/0x80 [99824.986665] ptlrpc_main+0xbec/0x1530 [ptlrpc] [99824.988003] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [99824.988998] kthread+0x134/0x150 [99824.989654] ? set_kthread_struct+0x50/0x50 [99824.990409] ret_from_fork+0x1f/0x40 [100156.922311] warn_alloc: 4 callbacks suppressed [100156.922314] ll_ost_io03_081: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100156.923458] ll_ost_io04_047: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100156.924199] ,cpuset= [100156.926809] ,cpuset= [100156.929351] / [100156.931027] / [100156.931027] ,mems_allowed=0 [100156.931730] ,mems_allowed=0 [100156.932391] CPU: 15 PID: 527554 Comm: ll_ost_io03_081 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100156.934201] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100156.935728] ll_ost_io01_027: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100156.942471] Call Trace: [100156.943327] dump_stack+0x41/0x60 [100156.943439] ll_ost_io01_031: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100156.944334] warn_alloc.cold.127+0x7b/0x108 [100156.947222] ll_ost_io02_000: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100156.947813] ll_ost_io04_064: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100156.948145] __alloc_pages_slowpath+0xcb2/0xcd0 [100156.949949] ,cpuset= [100156.952330] ? __alloc_pages_nodemask+0x166/0x330 [100156.953282] / [100156.953916] __alloc_pages_nodemask+0x2e2/0x330 [100156.954857] ,mems_allowed=0 [100156.955373] kmalloc_order+0x28/0x90 [100156.957012] kmalloc_order_trace+0x1d/0xb0 [100156.959162] __kmalloc+0x203/0x250 [100156.959905] virtqueue_add+0x493/0xc70 [100156.960699] ? finish_wait+0x80/0x80 [100156.961443] virtqueue_add_sgs+0x80/0xa0 [100156.962252] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100156.963385] ? scsi_alloc_sgtables+0x84/0x1c0 [100156.964516] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100156.965776] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100156.967076] scsi_queue_rq+0x512/0xb10 [100156.967853] __blk_mq_try_issue_directly+0x163/0x200 [100156.968795] blk_mq_request_issue_directly+0x4e/0xb0 [100156.969754] blk_mq_try_issue_list_directly+0x62/0x100 [100156.970880] blk_mq_sched_insert_requests+0xa4/0xf0 [100156.971857] blk_mq_flush_plug_list+0x135/0x220 [100156.972746] blk_flush_plug_list+0xd7/0x100 [100156.973577] blk_finish_plug+0x25/0x36 [100156.974430] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100156.975616] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100156.976724] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100156.977942] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100156.979277] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100156.980555] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100156.981604] ofd_commitrw+0x5b4/0xd20 [ofd] [100156.982664] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100156.983997] obd_commitrw+0x1b6/0x370 [ptlrpc] [100156.984984] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100156.985958] ? flush_work+0x42/0x1d0 [100156.986703] ? internal_add_timer+0x42/0x70 [100156.987518] ? _cond_resched+0x15/0x30 [100156.988304] ? mutex_lock+0xe/0x30 [100156.989007] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100156.990016] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100156.991072] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100156.992150] ? finish_wait+0x80/0x80 [100156.992865] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100156.993821] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100156.994872] kthread+0x134/0x150 [100156.995614] ? set_kthread_struct+0x50/0x50 [100156.996668] ret_from_fork+0x1f/0x40 [100156.997612] warn_alloc_show_mem: 9 callbacks suppressed [100156.997613] CPU: 7 PID: 527110 Comm: ll_ost_io01_027 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100156.997620] Mem-Info: [100156.998990] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100157.001682] active_anon:27909 inactive_anon:171493 isolated_anon:0 active_file:30249100 inactive_file:4717891 isolated_file:1 unevictable:0 dirty:188 writeback:1 slab_reclaimable:1009164 slab_unreclaimable:609841 mapped:20923 shmem:145 pagetables:5794 bounce:0 free:1230541 free_pcp:543 free_cma:0 [100157.002482] Call Trace: [100157.004655] Node 0 active_anon:111636kB inactive_anon:685972kB active_file:120996400kB inactive_file:18871564kB unevictable:0kB isolated(anon):0kB isolated(file):4kB mapped:83692kB dirty:752kB writeback:4kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35568kB pagetables:23176kB all_unreclaimable? no [100157.012998] dump_stack+0x41/0x60 [100157.013816] Node 0 [100157.020786] warn_alloc.cold.127+0x7b/0x108 [100157.021525] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100157.022308] __alloc_pages_slowpath+0xcb2/0xcd0 [100157.023138] lowmem_reserve[]: [100157.027827] ? __alloc_pages_nodemask+0x166/0x330 [100157.028804] 0 [100157.029487] __alloc_pages_nodemask+0x2e2/0x330 [100157.030396] 913 [100157.030927] kmalloc_order+0x28/0x90 [100157.031829] 150007 [100157.032375] kmalloc_order_trace+0x1d/0xb0 [100157.033165] 150007 [100157.033742] __kmalloc+0x203/0x250 [100157.034583] 150007 [100157.035138] virtqueue_add+0x493/0xc70 [100157.036457] ? finish_wait+0x80/0x80 [100157.037253] Node 0 [100157.037722] virtqueue_add_sgs+0x80/0xa0 [100157.038484] DMA32 free:596660kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:44kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100157.039033] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100157.039848] lowmem_reserve[]: [100157.043663] ? scsi_alloc_sgtables+0x84/0x1c0 [100157.044724] 0 [100157.045431] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100157.046306] 0 [100157.046819] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100157.047810] 149093 [100157.048325] scsi_queue_rq+0x512/0xb10 [100157.049402] 149093 [100157.050002] __blk_mq_try_issue_directly+0x163/0x200 [100157.050828] 149093 [100157.051406] blk_mq_request_issue_directly+0x4e/0xb0 [100157.052944] blk_mq_try_issue_list_directly+0x62/0x100 [100157.053894] Node 0 [100157.054339] blk_mq_sched_insert_requests+0xa4/0xf0 [100157.055311] Normal free:4312088kB min:49268kB low:201940kB high:354612kB active_anon:111256kB inactive_anon:685968kB active_file:121002848kB inactive_file:18867376kB unevictable:0kB writepending:756kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1936kB local_pcp:8kB free_cma:0kB [100157.055877] blk_mq_flush_plug_list+0x135/0x220 [100157.056811] lowmem_reserve[]: [100157.062425] blk_flush_plug_list+0xd7/0x100 [100157.063339] 0 [100157.064299] blk_finish_plug+0x25/0x36 [100157.065163] 0 [100157.065876] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100157.066654] 0 [100157.067350] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.068400] 0 [100157.069082] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.070213] 0 [100157.070754] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100157.072368] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100157.073324] Node 0 [100157.073824] ofd_commitrw+0x5b4/0xd20 [ofd] [100157.074746] DMA: [100157.075307] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.076121] 0*4kB [100157.076671] obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.077538] 0*8kB [100157.078091] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100157.078927] 0*16kB [100157.079550] ? flush_work+0x42/0x1d0 [100157.080431] 0*32kB [100157.081187] ? internal_add_timer+0x42/0x70 [100157.081933] 0*64kB [100157.082698] ? _cond_resched+0x15/0x30 [100157.083508] 0*128kB [100157.084285] ? mutex_lock+0xe/0x30 [100157.085065] 0*256kB [100157.085869] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100157.086551] 0*512kB [100157.087333] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100157.088245] 1*1024kB [100157.089036] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100157.090068] (U) [100157.090885] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100157.091906] 1*2048kB [100157.092623] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100157.093447] (M) [100157.094236] kthread+0x134/0x150 [100157.095187] 2*4096kB [100157.095910] ? set_kthread_struct+0x50/0x50 [100157.096603] (M) [100157.097389] ret_from_fork+0x1f/0x40 [100157.098214] = 11264kB [100157.098922] CPU: 4 PID: 527135 Comm: ll_ost_io01_031 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100157.099788] Node 0 [100157.100621] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100157.102995] DMA32: [100157.103848] Call Trace: [100157.105693] 17*4kB [100157.106501] dump_stack+0x41/0x60 [100157.107164] (UMH) [100157.107969] warn_alloc.cold.127+0x7b/0x108 [100157.108716] 26*8kB [100157.109404] __alloc_pages_slowpath+0xcb2/0xcd0 [100157.110317] (UMH) [100157.111031] ? __alloc_pages_nodemask+0x166/0x330 [100157.111941] 48*16kB [100157.112734] __alloc_pages_nodemask+0x2e2/0x330 [100157.113776] (UMEH) [100157.114568] kmalloc_order+0x28/0x90 [100157.115504] 47*32kB [100157.116267] kmalloc_order_trace+0x1d/0xb0 [100157.117086] (UMEH) [100157.117731] __kmalloc+0x203/0x250 [100157.118538] 41*64kB [100157.119176] ? __wake_up_common+0x7a/0x190 [100157.119878] (UMH) [100157.120649] virtqueue_add+0x493/0xc70 [100157.121448] 42*128kB [100157.122164] ? finish_wait+0x80/0x80 [100157.122912] (UM) [100157.123585] virtqueue_add_sgs+0x80/0xa0 [100157.124319] 42*256kB [100157.125055] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100157.125842] (UM) [100157.126630] ? scsi_alloc_sgtables+0x84/0x1c0 [100157.127633] 44*512kB [100157.128352] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100157.129203] (UME) [100157.129906] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100157.130853] 38*1024kB [100157.131585] scsi_queue_rq+0x512/0xb10 [100157.132603] (UM) [100157.133401] __blk_mq_try_issue_directly+0x163/0x200 [100157.134154] 41*2048kB [100157.134862] blk_mq_request_issue_directly+0x4e/0xb0 [100157.135782] (UM) [100157.136606] blk_mq_try_issue_list_directly+0x62/0x100 [100157.137540] 105*4096kB [100157.138253] blk_mq_sched_insert_requests+0xa4/0xf0 [100157.139203] (UM) [100157.140035] blk_mq_flush_plug_list+0x135/0x220 [100157.140940] = 596788kB [100157.141701] blk_flush_plug_list+0xd7/0x100 [100157.142560] Node 0 [100157.143376] blk_finish_plug+0x25/0x36 [100157.144160] Normal: [100157.144916] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100157.145648] 473302*4kB [100157.146394] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100157.147437] (UME) [100157.148175] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.149110] 274309*8kB [100157.149838] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.150940] (UME) [100157.151767] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100157.152817] 9074*16kB [100157.153539] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100157.154506] (UMEH) [100157.155122] ofd_commitrw+0x5b4/0xd20 [ofd] [100157.156011] 11*32kB [100157.156737] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.157523] (UH) [100157.158237] obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.159283] 1*64kB [100157.159975] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100157.160795] (H) [100157.161535] ? flush_work+0x42/0x1d0 [100157.162415] 0*128kB [100157.162959] ? internal_add_timer+0x42/0x70 [100157.163679] 0*256kB [100157.164249] ? _cond_resched+0x15/0x30 [100157.165061] 0*512kB [100157.165685] ? mutex_lock+0xe/0x30 [100157.166434] 0*1024kB [100157.166993] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100157.167711] 0*2048kB [100157.168433] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100157.169333] 0*4096kB [100157.169917] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100157.170943] = 4233280kB [100157.171521] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100157.172578] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100157.173228] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100157.174098] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100157.175604] kthread+0x134/0x150 [100157.176577] 34991015 total pagecache pages [100157.178039] ? set_kthread_struct+0x50/0x50 [100157.178769] 1312 pages in swap cache [100157.179602] ret_from_fork+0x1f/0x40 [100157.180454] Swap cache stats: add 1713873, delete 1712599, find 97953/118463 [100157.181213] CPU: 9 PID: 188103 Comm: ll_ost_io02_000 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100157.182013] Free swap = 5029396kB [100157.183487] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100157.185816] Total swap = 11075580kB [100157.186657] Call Trace: [100157.188289] 39321462 pages RAM [100157.189118] dump_stack+0x41/0x60 [100157.189767] 0 pages HighMem/MovableOnly [100157.190503] warn_alloc.cold.127+0x7b/0x108 [100157.191252] 905848 pages reserved [100157.192099] __alloc_pages_slowpath+0xcb2/0xcd0 [100157.192957] 0 pages hwpoisoned [100157.196227] __alloc_pages_nodemask+0x2e2/0x330 [100157.197367] kmalloc_order+0x28/0x90 [100157.198340] kmalloc_order_trace+0x1d/0xb0 [100157.199204] __kmalloc+0x203/0x250 [100157.199924] ? __wake_up_common+0x7a/0x190 [100157.200809] virtqueue_add+0x493/0xc70 [100157.201575] ? finish_wait+0x80/0x80 [100157.202465] virtqueue_add_sgs+0x80/0xa0 [100157.203332] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100157.204621] ? scsi_alloc_sgtables+0x84/0x1c0 [100157.205965] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100157.207234] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100157.208628] scsi_queue_rq+0x512/0xb10 [100157.209636] __blk_mq_try_issue_directly+0x163/0x200 [100157.210789] blk_mq_request_issue_directly+0x4e/0xb0 [100157.211759] blk_mq_try_issue_list_directly+0x62/0x100 [100157.212818] blk_mq_sched_insert_requests+0xa4/0xf0 [100157.214031] blk_mq_flush_plug_list+0x135/0x220 [100157.215193] blk_flush_plug_list+0xd7/0x100 [100157.216094] blk_finish_plug+0x25/0x36 [100157.216896] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100157.217943] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100157.218906] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.220003] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.221078] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100157.222033] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100157.222918] ofd_commitrw+0x5b4/0xd20 [ofd] [100157.223716] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.224649] obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.225552] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100157.226491] ? flush_work+0x42/0x1d0 [100157.227196] ? internal_add_timer+0x42/0x70 [100157.227966] ? _cond_resched+0x15/0x30 [100157.228692] ? mutex_lock+0xe/0x30 [100157.229360] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100157.230340] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100157.231451] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100157.232564] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100157.233471] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100157.234494] kthread+0x134/0x150 [100157.235152] ? set_kthread_struct+0x50/0x50 [100157.235925] ret_from_fork+0x1f/0x40 [100157.236617] CPU: 18 PID: 527997 Comm: ll_ost_io04_064 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100157.238901] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100157.240676] Call Trace: [100157.241208] dump_stack+0x41/0x60 [100157.241929] warn_alloc.cold.127+0x7b/0x108 [100157.242712] __alloc_pages_slowpath+0xcb2/0xcd0 [100157.243534] ? __alloc_pages_nodemask+0x166/0x330 [100157.244393] __alloc_pages_nodemask+0x2e2/0x330 [100157.245220] kmalloc_order+0x28/0x90 [100157.245932] kmalloc_order_trace+0x1d/0xb0 [100157.246715] __kmalloc+0x203/0x250 [100157.247389] virtqueue_add+0x493/0xc70 [100157.248116] ? finish_wait+0x80/0x80 [100157.248817] virtqueue_add_sgs+0x80/0xa0 [100157.249558] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100157.250542] ? scsi_alloc_sgtables+0x84/0x1c0 [100157.251360] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100157.252275] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100157.253253] scsi_queue_rq+0x512/0xb10 [100157.253975] __blk_mq_try_issue_directly+0x163/0x200 [100157.254871] blk_mq_request_issue_directly+0x4e/0xb0 [100157.255785] blk_mq_try_issue_list_directly+0x62/0x100 [100157.256724] blk_mq_sched_insert_requests+0xa4/0xf0 [100157.257597] blk_mq_flush_plug_list+0x135/0x220 [100157.258449] blk_flush_plug_list+0xd7/0x100 [100157.259240] blk_finish_plug+0x25/0x36 [100157.259961] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100157.260996] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100157.261949] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.263052] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.264134] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100157.265131] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100157.266037] ofd_commitrw+0x5b4/0xd20 [ofd] [100157.266836] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.267787] obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.268690] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100157.269647] ? flush_work+0x42/0x1d0 [100157.270381] ? internal_add_timer+0x42/0x70 [100157.271200] ? _cond_resched+0x15/0x30 [100157.271979] ? mutex_lock+0xe/0x30 [100157.272676] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100157.273706] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100157.274838] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100157.275934] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100157.276816] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100157.277791] kthread+0x134/0x150 [100157.278440] ? set_kthread_struct+0x50/0x50 [100157.279227] ret_from_fork+0x1f/0x40 [100157.279922] CPU: 16 PID: 527684 Comm: ll_ost_io04_047 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100157.282724] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100157.284809] Call Trace: [100157.285597] dump_stack+0x41/0x60 [100157.286528] warn_alloc.cold.127+0x7b/0x108 [100157.287636] __alloc_pages_slowpath+0xcb2/0xcd0 [100157.288808] ? __alloc_pages_nodemask+0x166/0x330 [100157.290698] __alloc_pages_nodemask+0x2e2/0x330 [100157.291890] kmalloc_order+0x28/0x90 [100157.292919] kmalloc_order_trace+0x1d/0xb0 [100157.294005] __kmalloc+0x203/0x250 [100157.294989] virtqueue_add+0x493/0xc70 [100157.296035] ? finish_wait+0x80/0x80 [100157.297042] virtqueue_add_sgs+0x80/0xa0 [100157.298118] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100157.299483] ? scsi_alloc_sgtables+0x84/0x1c0 [100157.300670] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100157.301991] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100157.303435] scsi_queue_rq+0x512/0xb10 [100157.304526] __blk_mq_try_issue_directly+0x163/0x200 [100157.305812] blk_mq_request_issue_directly+0x4e/0xb0 [100157.307112] blk_mq_try_issue_list_directly+0x62/0x100 [100157.308438] blk_mq_sched_insert_requests+0xa4/0xf0 [100157.309710] blk_mq_flush_plug_list+0x135/0x220 [100157.310964] blk_flush_plug_list+0xd7/0x100 [100157.312134] blk_finish_plug+0x25/0x36 [100157.313166] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100157.314586] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100157.315913] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.317487] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100157.319031] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100157.320373] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100157.321644] ofd_commitrw+0x5b4/0xd20 [ofd] [100157.322773] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.324096] obd_commitrw+0x1b6/0x370 [ptlrpc] [100157.325377] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100157.326697] ? newidle_balance+0x2b6/0x3b0 [100157.327760] ? internal_add_timer+0x42/0x70 [100157.328837] ? _cond_resched+0x15/0x30 [100157.329854] ? mutex_lock+0xe/0x30 [100157.330803] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100157.332210] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100157.333733] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100157.335288] ? finish_wait+0x80/0x80 [100157.336312] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100157.337645] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100157.339049] kthread+0x134/0x150 [100157.339968] ? set_kthread_struct+0x50/0x50 [100157.341078] ret_from_fork+0x1f/0x40 [100234.505168] ll_ost_io02_020: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100234.505179] ll_ost_io04_035: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100234.505188] ll_ost_io00_076: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100234.505188] ll_ost_io02_079: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [100234.505194] ,cpuset= [100234.505196] / [100234.505198] / [100234.505197] ,mems_allowed=0 [100234.505199] ,mems_allowed=0 [100234.505201] CPU: 8 PID: 527872 Comm: ll_ost_io02_079 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.505205] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.505207] Call Trace: [100234.505213] dump_stack+0x41/0x60 [100234.505222] warn_alloc.cold.127+0x7b/0x108 [100234.505229] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.505233] ? __alloc_pages_nodemask+0x166/0x330 [100234.505237] __alloc_pages_nodemask+0x2e2/0x330 [100234.505240] kmalloc_order+0x28/0x90 [100234.505246] kmalloc_order_trace+0x1d/0xb0 [100234.505250] __kmalloc+0x203/0x250 [100234.505256] virtqueue_add+0x493/0xc70 [100234.505263] virtqueue_add_sgs+0x80/0xa0 [100234.505268] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.505271] ll_ost_io02_014: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100234.505273] ll_ost_io02_070: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100234.505275] ,cpuset=/ [100234.505277] ,cpuset= [100234.505277] ,mems_allowed=0 [100234.505279] / [100234.505275] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.505280] ,mems_allowed=0 [100234.505282] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.505285] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.505289] scsi_queue_rq+0x512/0xb10 [100234.505292] __blk_mq_try_issue_directly+0x163/0x200 [100234.505298] blk_mq_request_issue_directly+0x4e/0xb0 [100234.505301] blk_mq_try_issue_list_directly+0x62/0x100 [100234.505305] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.505309] blk_mq_flush_plug_list+0x135/0x220 [100234.505313] blk_flush_plug_list+0xd7/0x100 [100234.505319] blk_finish_plug+0x25/0x36 [100234.505324] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.505361] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100234.505390] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.505414] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.505440] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.505466] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.505485] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.505501] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.505623] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.505729] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.505828] ? flush_work+0x42/0x1d0 [100234.505833] ? internal_add_timer+0x42/0x70 [100234.505836] ? _cond_resched+0x15/0x30 [100234.505839] ? mutex_lock+0xe/0x30 [100234.505842] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100234.505940] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100234.506044] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100234.506158] ? lu_context_exit+0x41/0x70 [obdclass] [100234.506240] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100234.506350] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100234.506469] kthread+0x134/0x150 [100234.506473] ? set_kthread_struct+0x50/0x50 [100234.506476] ret_from_fork+0x1f/0x40 [100234.506481] warn_alloc_show_mem: 5 callbacks suppressed [100234.506483] Mem-Info: [100234.506482] CPU: 11 PID: 527082 Comm: ll_ost_io02_014 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.506486] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.506488] Call Trace: [100234.506486] active_anon:27897 inactive_anon:167857 isolated_anon:0 active_file:28945570 inactive_file:6224068 isolated_file:9 unevictable:0 dirty:559 writeback:11 slab_reclaimable:1011416 slab_unreclaimable:619915 mapped:21721 shmem:145 pagetables:5797 bounce:0 free:1017167 free_pcp:530 free_cma:0 [100234.506492] Node 0 active_anon:111588kB inactive_anon:671428kB active_file:115782280kB inactive_file:24896272kB unevictable:0kB isolated(anon):0kB isolated(file):36kB mapped:86884kB dirty:2236kB writeback:44kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35600kB pagetables:23188kB all_unreclaimable? no [100234.506493] dump_stack+0x41/0x60 [100234.506499] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100234.506501] warn_alloc.cold.127+0x7b/0x108 [100234.506505] lowmem_reserve[]: 0 913 150007 150007 150007 [100234.506508] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.506511] Node 0 [100234.506512] ? __alloc_pages_nodemask+0x166/0x330 [100234.506512] DMA32 free:596496kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100234.506517] lowmem_reserve[]: 0 [100234.506517] __alloc_pages_nodemask+0x2e2/0x330 [100234.506519] 0 149093 149093 149093 [100234.506522] Node 0 [100234.506521] kmalloc_order+0x28/0x90 [100234.506523] Normal free:3460880kB min:51316kB low:203988kB high:356660kB active_anon:111208kB inactive_anon:671424kB active_file:115782216kB inactive_file:24896016kB unevictable:0kB writepending:2280kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2140kB local_pcp:136kB free_cma:0kB [100234.506526] kmalloc_order_trace+0x1d/0xb0 [100234.506529] lowmem_reserve[]: 0 0 0 0 0 [100234.506533] Node 0 [100234.506530] __kmalloc+0x203/0x250 [100234.506534] DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB [100234.506536] virtqueue_add+0x493/0xc70 [100234.506540] 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB [100234.506542] ? finish_wait+0x80/0x80 [100234.506545] (M) 2*4096kB (M) = 11264kB [100234.506547] virtqueue_add_sgs+0x80/0xa0 [100234.506549] Node 0 DMA32: 18*4kB (UM) 21*8kB (UMH) 46*16kB (UME) [100234.506553] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.506555] 41*32kB (UMEH) 40*64kB (UM) 42*128kB (UMH) 42*256kB (UMH) [100234.506559] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.506562] 44*512kB (UMEH) 38*1024kB (UMH) 41*2048kB [100234.506564] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.506566] (UM) 105*4096kB (UM) = 596464kB [100234.506570] Node 0 [100234.506569] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.506570] Normal: 493466*4kB (UME) 178051*8kB (UME) [100234.506573] scsi_queue_rq+0x512/0xb10 [100234.506575] 3966*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB [100234.506577] __blk_mq_try_issue_directly+0x163/0x200 [100234.506580] 0*512kB 0*1024kB 0*2048kB 0*4096kB = 3461728kB [100234.506584] blk_mq_request_issue_directly+0x4e/0xb0 [100234.506586] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100234.506588] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100234.506590] 35169313 total pagecache pages [100234.506589] blk_mq_try_issue_list_directly+0x62/0x100 [100234.506591] 1330 pages in swap cache [100234.506592] Swap cache stats: add 1719207, delete 1717903, find 98262/118839 [100234.506594] Free swap = 5010636kB [100234.506595] Total swap = 11075580kB [100234.506594] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.506596] 39321462 pages RAM [100234.506597] 0 pages HighMem/MovableOnly [100234.506598] 905848 pages reserved [100234.506598] 0 pages hwpoisoned [100234.506598] blk_mq_flush_plug_list+0x135/0x220 [100234.506603] blk_flush_plug_list+0xd7/0x100 [100234.506608] blk_finish_plug+0x25/0x36 [100234.506613] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.506651] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100234.506684] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.506714] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.506745] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.506778] ll_ost_io02_045: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100234.506776] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.506800] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.506819] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.506953] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.507064] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.507154] ? flush_work+0x42/0x1d0 [100234.507159] ? internal_add_timer+0x42/0x70 [100234.507163] ? _cond_resched+0x15/0x30 [100234.507165] ? mutex_lock+0xe/0x30 [100234.507169] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100234.507281] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100234.507408] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100234.507521] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100234.507641] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100234.507771] kthread+0x134/0x150 [100234.507775] ? set_kthread_struct+0x50/0x50 [100234.507778] ret_from_fork+0x1f/0x40 [100234.507784] CPU: 8 PID: 527250 Comm: ll_ost_io02_045 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.507788] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.507789] Call Trace: [100234.507792] dump_stack+0x41/0x60 [100234.507797] warn_alloc.cold.127+0x7b/0x108 [100234.507802] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.507806] ? __alloc_pages_nodemask+0x166/0x330 [100234.507809] __alloc_pages_nodemask+0x2e2/0x330 [100234.507812] kmalloc_order+0x28/0x90 [100234.507817] kmalloc_order_trace+0x1d/0xb0 [100234.507820] __kmalloc+0x203/0x250 [100234.507825] virtqueue_add+0x493/0xc70 [100234.507832] ? finish_wait+0x80/0x80 [100234.507835] virtqueue_add_sgs+0x80/0xa0 [100234.507839] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.507844] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.507849] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.507852] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.507855] scsi_queue_rq+0x512/0xb10 [100234.507858] __blk_mq_try_issue_directly+0x163/0x200 [100234.507864] blk_mq_request_issue_directly+0x4e/0xb0 [100234.507867] blk_mq_try_issue_list_directly+0x62/0x100 [100234.507871] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.507875] blk_mq_flush_plug_list+0x135/0x220 [100234.507879] blk_flush_plug_list+0xd7/0x100 [100234.507884] blk_finish_plug+0x25/0x36 [100234.507887] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.507928] ll_ost_io00_052: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [100234.507924] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.507935] /,mems_allowed=0 [100234.507949] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.507975] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.508001] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.508019] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.508035] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.508166] ll_ost_io02_021: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100234.508150] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.510806] ,cpuset= [100234.512965] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.516105] / [100234.516811] ? flush_work+0x42/0x1d0 [100234.517578] ,mems_allowed=0 [100234.518147] ? internal_add_timer+0x42/0x70 [100234.519578] ? _cond_resched+0x15/0x30 [100234.520932] ll_ost_io00_041: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100234.731192] ? mutex_lock+0xe/0x30 [100234.732003] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100234.733222] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100234.734554] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100234.735865] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100234.736932] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100234.738103] kthread+0x134/0x150 [100234.738867] ? set_kthread_struct+0x50/0x50 [100234.739776] ret_from_fork+0x1f/0x40 [100234.740624] CPU: 11 PID: 527175 Comm: ll_ost_io02_021 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.743067] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.744872] Call Trace: [100234.745546] dump_stack+0x41/0x60 [100234.746350] warn_alloc.cold.127+0x7b/0x108 [100234.747284] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.748307] ? __alloc_pages_nodemask+0x166/0x330 [100234.749342] __alloc_pages_nodemask+0x2e2/0x330 [100234.750355] kmalloc_order+0x28/0x90 [100234.751237] kmalloc_order_trace+0x1d/0xb0 [100234.752185] __kmalloc+0x203/0x250 [100234.753010] virtqueue_add+0x493/0xc70 [100234.753981] ? finish_wait+0x80/0x80 [100234.754958] virtqueue_add_sgs+0x80/0xa0 [100234.755870] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.757002] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.757966] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.759032] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.760192] scsi_queue_rq+0x512/0xb10 [100234.761042] __blk_mq_try_issue_directly+0x163/0x200 [100234.762090] blk_mq_request_issue_directly+0x4e/0xb0 [100234.763153] blk_mq_try_issue_list_directly+0x62/0x100 [100234.764229] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.765310] blk_mq_flush_plug_list+0x135/0x220 [100234.766272] blk_flush_plug_list+0xd7/0x100 [100234.767210] blk_finish_plug+0x25/0x36 [100234.768102] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.769352] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100234.770525] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.771873] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.773170] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.774286] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.775314] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.776238] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.777351] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.778440] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.779569] ? flush_work+0x42/0x1d0 [100234.780390] ? internal_add_timer+0x42/0x70 [100234.781303] ? _cond_resched+0x15/0x30 [100234.782161] ? mutex_lock+0xe/0x30 [100234.782975] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100234.784201] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100234.785537] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100234.786859] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100234.787948] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100234.789125] kthread+0x134/0x150 [100234.789907] ? set_kthread_struct+0x50/0x50 [100234.790823] ret_from_fork+0x1f/0x40 [100234.791646] CPU: 3 PID: 527429 Comm: ll_ost_io00_052 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.794203] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.796482] Call Trace: [100234.797151] dump_stack+0x41/0x60 [100234.797963] warn_alloc.cold.127+0x7b/0x108 [100234.799011] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.800081] ? __alloc_pages_nodemask+0x166/0x330 [100234.801177] __alloc_pages_nodemask+0x2e2/0x330 [100234.802229] kmalloc_order+0x28/0x90 [100234.803150] kmalloc_order_trace+0x1d/0xb0 [100234.804067] __kmalloc+0x203/0x250 [100234.804874] virtqueue_add+0x493/0xc70 [100234.805736] ? finish_wait+0x80/0x80 [100234.806556] virtqueue_add_sgs+0x80/0xa0 [100234.807433] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.808550] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.809507] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.810557] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.811728] scsi_queue_rq+0x512/0xb10 [100234.812587] __blk_mq_try_issue_directly+0x163/0x200 [100234.813638] blk_mq_request_issue_directly+0x4e/0xb0 [100234.814687] blk_mq_try_issue_list_directly+0x62/0x100 [100234.815757] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.816796] blk_mq_flush_plug_list+0x135/0x220 [100234.817771] blk_flush_plug_list+0xd7/0x100 [100234.818700] blk_finish_plug+0x25/0x36 [100234.819573] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.820821] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100234.821979] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.823292] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.824593] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.825739] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.826774] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.827704] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.828809] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.829886] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.831006] ? flush_work+0x42/0x1d0 [100234.831830] ? internal_add_timer+0x42/0x70 [100234.832757] ? _cond_resched+0x15/0x30 [100234.833616] ? mutex_lock+0xe/0x30 [100234.834403] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100234.835587] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100234.836891] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100234.838220] ? osd_key_exit+0x5/0x100 [osd_ldiskfs] [100234.839302] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100234.840375] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100234.841525] kthread+0x134/0x150 [100234.842287] ? set_kthread_struct+0x50/0x50 [100234.843190] ret_from_fork+0x1f/0x40 [100234.844006] CPU: 9 PID: 527735 Comm: ll_ost_io02_070 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.846369] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.848037] Call Trace: [100234.848812] dump_stack+0x41/0x60 [100234.849767] warn_alloc.cold.127+0x7b/0x108 [100234.850864] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.852022] ? __alloc_pages_nodemask+0x166/0x330 [100234.853208] __alloc_pages_nodemask+0x2e2/0x330 [100234.854375] kmalloc_order+0x28/0x90 [100234.855352] kmalloc_order_trace+0x1d/0xb0 [100234.856417] __kmalloc+0x203/0x250 [100234.857339] ? __wake_up_common+0x7a/0x190 [100234.858385] virtqueue_add+0x493/0xc70 [100234.859370] ? finish_wait+0x80/0x80 [100234.860257] virtqueue_add_sgs+0x80/0xa0 [100234.861195] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.862288] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.863273] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.864359] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.865472] scsi_queue_rq+0x512/0xb10 [100234.866258] __blk_mq_try_issue_directly+0x163/0x200 [100234.867241] blk_mq_request_issue_directly+0x4e/0xb0 [100234.868497] blk_mq_try_issue_list_directly+0x62/0x100 [100234.869798] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.870859] blk_mq_flush_plug_list+0x135/0x220 [100234.871730] blk_flush_plug_list+0xd7/0x100 [100234.872524] blk_finish_plug+0x25/0x36 [100234.873251] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.874286] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100234.875234] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.876404] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.877920] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.879237] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.880444] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.881539] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.882861] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.883954] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.885279] ? flush_work+0x42/0x1d0 [100234.886254] ? internal_add_timer+0x42/0x70 [100234.887067] ? _cond_resched+0x15/0x30 [100234.888035] ? mutex_lock+0xe/0x30 [100234.888988] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100234.890407] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100234.891956] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100234.893369] ? finish_wait+0x80/0x80 [100234.894113] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100234.895020] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100234.896023] kthread+0x134/0x150 [100234.896705] ? set_kthread_struct+0x50/0x50 [100234.897519] ret_from_fork+0x1f/0x40 [100234.898242] CPU: 16 PID: 527352 Comm: ll_ost_io04_035 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.900358] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.901919] Call Trace: [100234.902498] dump_stack+0x41/0x60 [100234.903193] warn_alloc.cold.127+0x7b/0x108 [100234.903999] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.904868] ? __alloc_pages_nodemask+0x166/0x330 [100234.905782] __alloc_pages_nodemask+0x2e2/0x330 [100234.906732] kmalloc_order+0x28/0x90 [100234.907610] kmalloc_order_trace+0x1d/0xb0 [100234.908672] __kmalloc+0x203/0x250 [100234.909602] virtqueue_add+0x493/0xc70 [100234.910604] ? finish_wait+0x80/0x80 [100234.911587] virtqueue_add_sgs+0x80/0xa0 [100234.912617] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.913897] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.915017] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.916266] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.917625] scsi_queue_rq+0x512/0xb10 [100234.918616] __blk_mq_try_issue_directly+0x163/0x200 [100234.919847] blk_mq_request_issue_directly+0x4e/0xb0 [100234.921063] blk_mq_try_issue_list_directly+0x62/0x100 [100234.922299] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.923493] blk_mq_flush_plug_list+0x135/0x220 [100234.924608] blk_flush_plug_list+0xd7/0x100 [100234.925572] blk_finish_plug+0x25/0x36 [100234.926327] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.927408] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100234.928448] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.929931] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.931325] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.932485] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.933607] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.934552] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.935703] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.936750] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.937899] ? flush_work+0x42/0x1d0 [100234.938890] ? internal_add_timer+0x42/0x70 [100234.939996] ? _cond_resched+0x15/0x30 [100234.941009] ? mutex_lock+0xe/0x30 [100234.941949] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100234.943137] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100234.944368] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100234.945631] ? tgt_key_exit+0xc/0x20 [ptlrpc] [100234.946863] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100234.948108] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100234.949259] kthread+0x134/0x150 [100234.950157] ? set_kthread_struct+0x50/0x50 [100234.951179] ret_from_fork+0x1f/0x40 [100234.951951] CPU: 10 PID: 527173 Comm: ll_ost_io02_020 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100234.954768] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100234.956727] Call Trace: [100234.957365] dump_stack+0x41/0x60 [100234.958338] warn_alloc.cold.127+0x7b/0x108 [100234.959475] __alloc_pages_slowpath+0xcb2/0xcd0 [100234.960689] ? __alloc_pages_nodemask+0x166/0x330 [100234.961883] __alloc_pages_nodemask+0x2e2/0x330 [100234.963051] kmalloc_order+0x28/0x90 [100234.964083] kmalloc_order_trace+0x1d/0xb0 [100234.965229] __kmalloc+0x203/0x250 [100234.966229] virtqueue_add+0x493/0xc70 [100234.967317] ? finish_wait+0x80/0x80 [100234.968369] virtqueue_add_sgs+0x80/0xa0 [100234.969429] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100234.970811] ? scsi_alloc_sgtables+0x84/0x1c0 [100234.971999] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100234.973300] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100234.974668] scsi_queue_rq+0x512/0xb10 [100234.975694] __blk_mq_try_issue_directly+0x163/0x200 [100234.976944] blk_mq_request_issue_directly+0x4e/0xb0 [100234.978201] blk_mq_try_issue_list_directly+0x62/0x100 [100234.979512] blk_mq_sched_insert_requests+0xa4/0xf0 [100234.980802] blk_mq_flush_plug_list+0x135/0x220 [100234.982042] blk_flush_plug_list+0xd7/0x100 [100234.983185] blk_finish_plug+0x25/0x36 [100234.984256] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100234.985759] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100234.987127] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.988722] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100234.990298] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100234.991656] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100234.992940] ofd_commitrw+0x5b4/0xd20 [ofd] [100234.994084] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.995407] obd_commitrw+0x1b6/0x370 [ptlrpc] [100234.996705] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100234.998184] ? flush_work+0x42/0x1d0 [100234.999184] ? internal_add_timer+0x42/0x70 [100235.000279] ? _cond_resched+0x15/0x30 [100235.001318] ? mutex_lock+0xe/0x30 [100235.002279] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100235.003706] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100235.005300] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100235.006860] ? lu_context_exit+0x2f/0x70 [obdclass] [100235.008199] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100235.009494] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100235.010884] kthread+0x134/0x150 [100235.011827] ? set_kthread_struct+0x50/0x50 [100235.012932] ret_from_fork+0x1f/0x40 [100235.013914] CPU: 0 PID: 527781 Comm: ll_ost_io00_076 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100235.016284] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100235.018329] Call Trace: [100235.019113] dump_stack+0x41/0x60 [100235.020026] warn_alloc.cold.127+0x7b/0x108 [100235.021114] __alloc_pages_slowpath+0xcb2/0xcd0 [100235.022267] ? __alloc_pages_nodemask+0x166/0x330 [100235.023456] __alloc_pages_nodemask+0x2e2/0x330 [100235.024509] kmalloc_order+0x28/0x90 [100235.025295] kmalloc_order_trace+0x1d/0xb0 [100235.026345] __kmalloc+0x203/0x250 [100235.027035] virtqueue_add+0x493/0xc70 [100235.027753] ? finish_wait+0x80/0x80 [100235.028440] virtqueue_add_sgs+0x80/0xa0 [100235.029254] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100235.030523] ? scsi_alloc_sgtables+0x84/0x1c0 [100235.031636] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100235.032881] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100235.034243] scsi_queue_rq+0x512/0xb10 [100235.035252] __blk_mq_try_issue_directly+0x163/0x200 [100235.036481] blk_mq_request_issue_directly+0x4e/0xb0 [100235.037719] blk_mq_try_issue_list_directly+0x62/0x100 [100235.038975] blk_mq_sched_insert_requests+0xa4/0xf0 [100235.040184] blk_mq_flush_plug_list+0x135/0x220 [100235.041319] blk_flush_plug_list+0xd7/0x100 [100235.042123] blk_finish_plug+0x25/0x36 [100235.042868] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100235.043897] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100235.044833] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100235.045958] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100235.047453] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100235.048781] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100235.050009] ofd_commitrw+0x5b4/0xd20 [ofd] [100235.051118] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100235.052441] obd_commitrw+0x1b6/0x370 [ptlrpc] [100235.053708] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100235.055018] ? flush_work+0x42/0x1d0 [100235.055992] ? internal_add_timer+0x42/0x70 [100235.057063] ? _cond_resched+0x15/0x30 [100235.058053] ? mutex_lock+0xe/0x30 [100235.058985] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100235.060368] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100235.061872] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100235.063270] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100235.064390] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100235.065732] kthread+0x134/0x150 [100235.066618] ? set_kthread_struct+0x50/0x50 [100235.067682] ret_from_fork+0x1f/0x40 [100235.068640] CPU: 2 PID: 527355 Comm: ll_ost_io00_041 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100235.071500] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100235.073591] Call Trace: [100235.074394] dump_stack+0x41/0x60 [100235.075320] warn_alloc.cold.127+0x7b/0x108 [100235.076476] __alloc_pages_slowpath+0xcb2/0xcd0 [100235.077688] __alloc_pages_nodemask+0x2e2/0x330 [100235.078916] kmalloc_order+0x28/0x90 [100235.079930] kmalloc_order_trace+0x1d/0xb0 [100235.081069] __kmalloc+0x203/0x250 [100235.082070] virtqueue_add+0x493/0xc70 [100235.083158] virtqueue_add_sgs+0x80/0xa0 [100235.084275] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100235.085674] ? scsi_alloc_sgtables+0x84/0x1c0 [100235.086871] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100235.088212] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100235.089682] scsi_queue_rq+0x512/0xb10 [100235.090764] __blk_mq_try_issue_directly+0x163/0x200 [100235.092080] blk_mq_request_issue_directly+0x4e/0xb0 [100235.093409] blk_mq_try_issue_list_directly+0x62/0x100 [100235.094772] blk_mq_sched_insert_requests+0xa4/0xf0 [100235.096045] blk_mq_flush_plug_list+0x135/0x220 [100235.097314] blk_flush_plug_list+0xd7/0x100 [100235.098496] blk_finish_plug+0x25/0x36 [100235.099552] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100235.101121] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100235.102571] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100235.104142] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100235.105646] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100235.106996] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100235.108110] ofd_commitrw+0x5b4/0xd20 [ofd] [100235.109117] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100235.110343] obd_commitrw+0x1b6/0x370 [ptlrpc] [100235.111668] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100235.112903] ? flush_work+0x42/0x1d0 [100235.113847] ? internal_add_timer+0x42/0x70 [100235.114974] ? _cond_resched+0x15/0x30 [100235.116008] ? mutex_lock+0xe/0x30 [100235.116983] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100235.118346] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100235.119494] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100235.120657] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100235.121603] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100235.122645] kthread+0x134/0x150 [100235.123341] ? set_kthread_struct+0x50/0x50 [100235.124164] ret_from_fork+0x1f/0x40 [100239.564651] warn_alloc: 1007 callbacks suppressed [100239.564655] ll_ost_io04_053: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100239.565952] ll_ost_io02_055: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100239.566183] ,cpuset= [100239.568801] ,cpuset= [100239.571432] / [100239.573233] / [100239.573233] ,mems_allowed=0 [100239.573238] CPU: 19 PID: 527690 Comm: ll_ost_io04_053 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100239.573966] ,mems_allowed=0 [100239.574707] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100239.578622] Call Trace: [100239.581804] ll_ost_io03_030: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100239.581923] dump_stack+0x41/0x60 [100239.582662] ,cpuset= [100239.583702] warn_alloc.cold.127+0x7b/0x108 [100239.586123] / [100239.587260] __alloc_pages_slowpath+0xcb2/0xcd0 [100239.588125] ,mems_allowed=0 [100239.589416] ? __alloc_pages_nodemask+0x166/0x330 [100239.589747] ll_ost_io04_073: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100239.590753] ll_ost_io02_059: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100239.591460] __alloc_pages_nodemask+0x2e2/0x330 [100239.595197] ll_ost_io03_016: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100239.596520] kmalloc_order+0x28/0x90 [100239.597149] ,cpuset= [100239.599980] kmalloc_order_trace+0x1d/0xb0 [100239.601112] / [100239.603534] __kmalloc+0x203/0x250 [100239.604492] ,mems_allowed=0 [100239.605119] ? __wake_up_common+0x7a/0x190 [100239.606829] virtqueue_add+0x493/0xc70 [100239.609247] ll_ost_io02_071: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100239.609251] ll_ost_io04_033: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100239.609257] ll_ost_io02_022: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100239.609271] ll_ost_io01_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100239.609841] ? finish_wait+0x80/0x80 [100239.610432] ,cpuset= [100239.611574] virtqueue_add_sgs+0x80/0xa0 [100239.613606] / [100239.615744] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100239.618061] ,mems_allowed=0 [100239.620164] ? scsi_alloc_sgtables+0x84/0x1c0 [100239.621693] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100239.628656] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100239.630126] scsi_queue_rq+0x512/0xb10 [100239.631233] __blk_mq_try_issue_directly+0x163/0x200 [100239.632537] blk_mq_request_issue_directly+0x4e/0xb0 [100239.633842] blk_mq_try_issue_list_directly+0x62/0x100 [100239.635174] blk_mq_sched_insert_requests+0xa4/0xf0 [100239.636473] blk_mq_flush_plug_list+0x135/0x220 [100239.637754] blk_flush_plug_list+0xd7/0x100 [100239.638958] blk_finish_plug+0x25/0x36 [100239.640064] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100239.641617] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100239.643071] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.644650] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.645796] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100239.646788] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100239.647698] ofd_commitrw+0x5b4/0xd20 [ofd] [100239.648568] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.649604] obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.650571] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100239.651570] ? flush_work+0x42/0x1d0 [100239.652301] ? internal_add_timer+0x42/0x70 [100239.653097] ? _cond_resched+0x15/0x30 [100239.653846] ? mutex_lock+0xe/0x30 [100239.654566] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100239.655582] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100239.656664] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100239.657786] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100239.658710] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100239.659697] kthread+0x134/0x150 [100239.660346] ? set_kthread_struct+0x50/0x50 [100239.661103] ret_from_fork+0x1f/0x40 [100239.661815] warn_alloc_show_mem: 9 callbacks suppressed [100239.661817] Mem-Info: [100239.661815] CPU: 15 PID: 527055 Comm: ll_ost_io03_016 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100239.662774] active_anon:27897 inactive_anon:167558 isolated_anon:0 active_file:28465300 inactive_file:4724327 isolated_file:0 unevictable:0 dirty:67821 writeback:0 slab_reclaimable:1026222 slab_unreclaimable:617773 mapped:22108 shmem:145 pagetables:5798 bounce:0 free:2985841 free_pcp:478 free_cma:0 [100239.663443] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100239.665579] Node 0 active_anon:111588kB inactive_anon:670260kB active_file:113865256kB inactive_file:18897336kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:88432kB dirty:271284kB writeback:0kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35600kB pagetables:23192kB all_unreclaimable? no [100239.672487] Call Trace: [100239.674044] Node 0 [100239.679857] dump_stack+0x41/0x60 [100239.680491] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100239.681151] warn_alloc.cold.127+0x7b/0x108 [100239.681875] lowmem_reserve[]: [100239.686080] __alloc_pages_slowpath+0xcb2/0xcd0 [100239.686945] 0 [100239.687738] ? __alloc_pages_nodemask+0x2e2/0x330 [100239.688641] 913 [100239.689235] __alloc_pages_nodemask+0x2e2/0x330 [100239.690161] 150007 [100239.690771] kmalloc_order+0x28/0x90 [100239.691681] 150007 [100239.692334] kmalloc_order_trace+0x1d/0xb0 [100239.693103] 150007 [100239.693755] __kmalloc+0x203/0x250 [100239.695273] virtqueue_add+0x493/0xc70 [100239.696014] Node 0 [100239.696564] ? finish_wait+0x80/0x80 [100239.697382] DMA32 free:596608kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100239.698052] virtqueue_add_sgs+0x80/0xa0 [100239.698849] lowmem_reserve[]: [100239.703307] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100239.704152] 0 [100239.705000] ? scsi_alloc_sgtables+0x84/0x1c0 [100239.706084] 0 [100239.706741] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100239.707689] 149093 [100239.708358] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100239.709398] 149093 [100239.710117] scsi_queue_rq+0x512/0xb10 [100239.711235] 149093 [100239.711926] __blk_mq_try_issue_directly+0x163/0x200 [100239.713435] blk_mq_request_issue_directly+0x4e/0xb0 [100239.714466] Node 0 [100239.715063] blk_mq_try_issue_list_directly+0x62/0x100 [100239.716048] Normal free:11307076kB min:49268kB low:201940kB high:354612kB active_anon:111208kB inactive_anon:670256kB active_file:113893732kB inactive_file:18893196kB unevictable:0kB writepending:271500kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1740kB local_pcp:152kB free_cma:0kB [100239.716710] blk_mq_sched_insert_requests+0xa4/0xf0 [100239.717716] lowmem_reserve[]: [100239.722786] blk_mq_flush_plug_list+0x135/0x220 [100239.723799] 0 [100239.724650] blk_flush_plug_list+0xd7/0x100 [100239.725596] 0 [100239.726169] blk_finish_plug+0x25/0x36 [100239.727168] 0 [100239.727752] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100239.728553] 0 [100239.729156] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100239.730428] 0 [100239.731032] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.732640] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.733821] Node 0 [100239.734362] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100239.735499] DMA: [100239.736154] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100239.737155] 0*4kB [100239.737781] ofd_commitrw+0x5b4/0xd20 [ofd] [100239.738735] 0*8kB [100239.739349] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.740550] 0*16kB [100239.741208] obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.742467] 0*32kB [100239.743144] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100239.744347] 0*64kB [100239.745034] ? newidle_balance+0x2b6/0x3b0 [100239.746346] 0*128kB [100239.747030] ? internal_add_timer+0x42/0x70 [100239.747948] 0*256kB [100239.748612] ? _cond_resched+0x15/0x30 [100239.749480] 0*512kB [100239.750141] ? mutex_lock+0xe/0x30 [100239.750948] 1*1024kB [100239.751579] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100239.752720] (U) [100239.753357] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100239.754738] 1*2048kB [100239.755327] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100239.756406] (M) [100239.757095] ? finish_wait+0x80/0x80 [100239.759335] 2*4096kB [100239.759946] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100239.760701] (M) [100239.761379] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100239.762347] = 11264kB [100239.762967] kthread+0x134/0x150 [100239.763967] Node 0 [100239.764645] ? set_kthread_struct+0x50/0x50 [100239.765376] DMA32: [100239.766028] ret_from_fork+0x1f/0x40 [100239.766876] 18*4kB [100239.767517] CPU: 16 PID: 527279 Comm: ll_ost_io04_033 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100239.768244] (UM) [100239.768909] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100239.771049] 21*8kB [100239.771678] Call Trace: [100239.773242] (UMH) [100239.773890] dump_stack+0x41/0x60 [100239.774486] 47*16kB [100239.775113] warn_alloc.cold.127+0x7b/0x108 [100239.775840] (UME) [100239.776491] __alloc_pages_slowpath+0xcb2/0xcd0 [100239.777354] 50*32kB [100239.777988] ? __alloc_pages_nodemask+0x166/0x330 [100239.778883] (UMEH) [100239.779526] __alloc_pages_nodemask+0x2e2/0x330 [100239.780432] 40*64kB [100239.781068] kmalloc_order+0x28/0x90 [100239.781947] (UM) [100239.782579] kmalloc_order_trace+0x1d/0xb0 [100239.783325] 42*128kB [100239.783949] __kmalloc+0x203/0x250 [100239.784788] (UMH) [100239.785453] virtqueue_add+0x493/0xc70 [100239.786187] 42*256kB [100239.786796] ? finish_wait+0x80/0x80 [100239.787569] (UMH) [100239.788225] virtqueue_add_sgs+0x80/0xa0 [100239.788977] 44*512kB [100239.789596] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100239.790397] (UMEH) [100239.791065] ? scsi_alloc_sgtables+0x84/0x1c0 [100239.792067] 38*1024kB [100239.792708] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100239.793583] (UMH) [100239.794260] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100239.795228] 41*2048kB [100239.795851] scsi_queue_rq+0x512/0xb10 [100239.796906] (UM) [100239.797591] __blk_mq_try_issue_directly+0x163/0x200 [100239.798387] 105*4096kB [100239.799010] blk_mq_request_issue_directly+0x4e/0xb0 [100239.799955] (UM) [100239.800624] blk_mq_try_issue_list_directly+0x62/0x100 [100239.801564] = 596768kB [100239.802175] blk_mq_sched_insert_requests+0xa4/0xf0 [100239.803165] Node 0 [100239.803839] blk_mq_flush_plug_list+0x135/0x220 [100239.804780] Normal: [100239.805412] blk_flush_plug_list+0xd7/0x100 [100239.806302] 1956266*4kB [100239.806943] blk_finish_plug+0x25/0x36 [100239.807787] (UM) [100239.808467] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100239.809258] 469367*8kB [100239.809851] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100239.810907] (UE) [100239.811559] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.812515] 16894*16kB [100239.813089] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.814242] (UEH) [100239.814891] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100239.816000] 1*32kB [100239.816587] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100239.817573] (H) [100239.818220] ofd_commitrw+0x5b4/0xd20 [ofd] [100239.819142] 1*64kB [100239.819725] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.820658] (H) [100239.821268] obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.822166] 0*128kB [100239.822737] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100239.823599] 0*256kB [100239.824259] ? flush_work+0x42/0x1d0 [100239.825164] 0*512kB [100239.825796] ? internal_add_timer+0x42/0x70 [100239.826544] 0*1024kB [100239.827198] ? _cond_resched+0x15/0x30 [100239.828035] 0*2048kB [100239.828693] ? mutex_lock+0xe/0x30 [100239.829471] 0*4096kB [100239.830131] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100239.830841] = 11850400kB [100239.831493] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100239.832433] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100239.833125] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100239.834154] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100239.835867] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100239.836940] 33058508 total pagecache pages [100239.838628] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100239.839504] 1327 pages in swap cache [100239.840478] kthread+0x134/0x150 [100239.841461] Swap cache stats: add 1719775, delete 1718474, find 98360/118951 [100239.842355] ? set_kthread_struct+0x50/0x50 [100239.843065] Free swap = 5009604kB [100239.844516] ret_from_fork+0x1f/0x40 [100239.845367] Total swap = 11075580kB [100239.846208] CPU: 11 PID: 527334 Comm: ll_ost_io02_059 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100239.847126] 39321462 pages RAM [100239.848032] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100239.850155] 0 pages HighMem/MovableOnly [100239.850986] Call Trace: [100239.852566] 905848 pages reserved [100239.853539] dump_stack+0x41/0x60 [100239.854152] 0 pages hwpoisoned [100239.855011] warn_alloc.cold.127+0x7b/0x108 [100239.860991] __alloc_pages_slowpath+0xcb2/0xcd0 [100239.867722] __alloc_pages_nodemask+0x2e2/0x330 [100239.869927] kmalloc_order+0x28/0x90 [100239.870788] kmalloc_order_trace+0x1d/0xb0 [100239.871757] __kmalloc+0x203/0x250 [100239.872567] virtqueue_add+0x493/0xc70 [100239.873417] virtqueue_add_sgs+0x80/0xa0 [100239.874314] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100239.875453] ? scsi_alloc_sgtables+0x84/0x1c0 [100239.876420] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100239.877490] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100239.878655] scsi_queue_rq+0x512/0xb10 [100239.879490] __blk_mq_try_issue_directly+0x163/0x200 [100239.880536] blk_mq_request_issue_directly+0x4e/0xb0 [100239.881585] blk_mq_try_issue_list_directly+0x62/0x100 [100239.882672] blk_mq_sched_insert_requests+0xa4/0xf0 [100239.883711] blk_mq_flush_plug_list+0x135/0x220 [100239.884704] blk_flush_plug_list+0xd7/0x100 [100239.885639] blk_finish_plug+0x25/0x36 [100239.886525] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100239.887817] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100239.888990] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.890344] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.891648] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100239.892791] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100239.893845] ofd_commitrw+0x5b4/0xd20 [ofd] [100239.894812] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.895976] obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.897108] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100239.898233] ? flush_work+0x42/0x1d0 [100239.899071] ? internal_add_timer+0x42/0x70 [100239.900004] ? _cond_resched+0x15/0x30 [100239.900854] ? mutex_lock+0xe/0x30 [100239.901646] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100239.902813] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100239.904101] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100239.905401] ? tgt_mask_cksum_types+0x70/0x70 [ptlrpc] [100239.906584] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100239.907660] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100239.908826] kthread+0x134/0x150 [100239.909605] ? set_kthread_struct+0x50/0x50 [100239.910544] ret_from_fork+0x1f/0x40 [100239.911384] CPU: 8 PID: 527736 Comm: ll_ost_io02_071 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100239.913780] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100239.915498] Call Trace: [100239.916132] dump_stack+0x41/0x60 [100239.916910] warn_alloc.cold.127+0x7b/0x108 [100239.917856] __alloc_pages_slowpath+0xcb2/0xcd0 [100239.918850] ? __alloc_pages_nodemask+0x166/0x330 [100239.919873] __alloc_pages_nodemask+0x2e2/0x330 [100239.920875] kmalloc_order+0x28/0x90 [100239.921706] kmalloc_order_trace+0x1d/0xb0 [100239.922611] __kmalloc+0x203/0x250 [100239.923403] virtqueue_add+0x493/0xc70 [100239.924227] ? finish_wait+0x80/0x80 [100239.925026] virtqueue_add_sgs+0x80/0xa0 [100239.925877] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100239.926969] ? scsi_alloc_sgtables+0x84/0x1c0 [100239.927916] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100239.928948] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100239.930087] scsi_queue_rq+0x512/0xb10 [100239.930942] __blk_mq_try_issue_directly+0x163/0x200 [100239.931980] blk_mq_request_issue_directly+0x4e/0xb0 [100239.933008] blk_mq_try_issue_list_directly+0x62/0x100 [100239.934065] blk_mq_sched_insert_requests+0xa4/0xf0 [100239.935088] blk_mq_flush_plug_list+0x135/0x220 [100239.936064] blk_flush_plug_list+0xd7/0x100 [100239.936993] blk_finish_plug+0x25/0x36 [100239.937840] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100239.939051] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100239.940157] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.941429] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.942669] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100239.943764] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100239.944797] ofd_commitrw+0x5b4/0xd20 [ofd] [100239.945729] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.946841] obd_commitrw+0x1b6/0x370 [ptlrpc] [100239.947903] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100239.948991] ? flush_work+0x42/0x1d0 [100239.949807] ? internal_add_timer+0x42/0x70 [100239.950699] ? _cond_resched+0x15/0x30 [100239.951542] ? mutex_lock+0xe/0x30 [100239.952327] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100239.953468] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100239.954717] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100239.956036] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100239.957124] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100239.958308] kthread+0x134/0x150 [100239.959081] ? set_kthread_struct+0x50/0x50 [100239.959979] ret_from_fork+0x1f/0x40 [100239.960802] CPU: 14 PID: 527230 Comm: ll_ost_io03_030 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100239.963592] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100239.965683] Call Trace: [100239.966447] dump_stack+0x41/0x60 [100239.967361] warn_alloc.cold.127+0x7b/0x108 [100239.968467] __alloc_pages_slowpath+0xcb2/0xcd0 [100239.969671] ? __alloc_pages_nodemask+0x166/0x330 [100239.970863] __alloc_pages_nodemask+0x2e2/0x330 [100239.971984] kmalloc_order+0x28/0x90 [100239.972931] kmalloc_order_trace+0x1d/0xb0 [100239.973973] __kmalloc+0x203/0x250 [100239.974894] ? __wake_up_common+0x7a/0x190 [100239.975939] virtqueue_add+0x493/0xc70 [100239.976958] ? finish_wait+0x80/0x80 [100239.977935] virtqueue_add_sgs+0x80/0xa0 [100239.978964] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100239.980294] ? scsi_alloc_sgtables+0x84/0x1c0 [100239.981401] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100239.982645] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100239.983719] scsi_queue_rq+0x512/0xb10 [100239.984430] __blk_mq_try_issue_directly+0x163/0x200 [100239.985319] blk_mq_request_issue_directly+0x4e/0xb0 [100239.986245] blk_mq_try_issue_list_directly+0x62/0x100 [100239.987447] blk_mq_sched_insert_requests+0xa4/0xf0 [100239.988649] blk_mq_flush_plug_list+0x135/0x220 [100239.989790] blk_flush_plug_list+0xd7/0x100 [100239.990856] blk_finish_plug+0x25/0x36 [100239.991847] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100239.993280] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100239.994584] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.996098] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100239.997581] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100239.998879] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100240.000093] ofd_commitrw+0x5b4/0xd20 [ofd] [100240.001189] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.002483] obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.003736] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100240.005030] ? set_next_entity+0xb5/0x1e0 [100240.006114] ? internal_add_timer+0x42/0x70 [100240.007203] ? _cond_resched+0x15/0x30 [100240.008180] ? mutex_lock+0xe/0x30 [100240.009136] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100240.010512] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100240.012016] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100240.013516] ? finish_wait+0x80/0x80 [100240.014466] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100240.015705] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100240.017058] kthread+0x134/0x150 [100240.017975] ? set_kthread_struct+0x50/0x50 [100240.019059] ret_from_fork+0x1f/0x40 [100240.020010] CPU: 17 PID: 528061 Comm: ll_ost_io04_073 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100240.022586] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100240.024760] Call Trace: [100240.025576] dump_stack+0x41/0x60 [100240.026534] warn_alloc.cold.127+0x7b/0x108 [100240.027688] __alloc_pages_slowpath+0xcb2/0xcd0 [100240.028950] ? __alloc_pages_nodemask+0x166/0x330 [100240.030238] __alloc_pages_nodemask+0x2e2/0x330 [100240.031506] kmalloc_order+0x28/0x90 [100240.032558] kmalloc_order_trace+0x1d/0xb0 [100240.033719] __kmalloc+0x203/0x250 [100240.034766] virtqueue_add+0x493/0xc70 [100240.035880] ? finish_wait+0x80/0x80 [100240.036980] virtqueue_add_sgs+0x80/0xa0 [100240.038099] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100240.039542] ? scsi_alloc_sgtables+0x84/0x1c0 [100240.040772] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100240.042121] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100240.043673] scsi_queue_rq+0x512/0xb10 [100240.044817] __blk_mq_try_issue_directly+0x163/0x200 [100240.046141] blk_mq_request_issue_directly+0x4e/0xb0 [100240.047406] blk_mq_try_issue_list_directly+0x62/0x100 [100240.048750] blk_mq_sched_insert_requests+0xa4/0xf0 [100240.049974] blk_mq_flush_plug_list+0x135/0x220 [100240.051134] blk_flush_plug_list+0xd7/0x100 [100240.052254] blk_finish_plug+0x25/0x36 [100240.053314] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100240.054794] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100240.056144] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.057700] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.059247] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100240.060578] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100240.061833] ofd_commitrw+0x5b4/0xd20 [ofd] [100240.062957] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.064311] obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.065640] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100240.067031] ? flush_work+0x42/0x1d0 [100240.068046] ? internal_add_timer+0x42/0x70 [100240.069174] ? _cond_resched+0x15/0x30 [100240.070189] ? mutex_lock+0xe/0x30 [100240.071168] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100240.072592] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100240.074073] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100240.075564] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100240.076779] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100240.078217] kthread+0x134/0x150 [100240.079190] ? set_kthread_struct+0x50/0x50 [100240.080333] ret_from_fork+0x1f/0x40 [100240.081359] CPU: 10 PID: 527177 Comm: ll_ost_io02_022 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100240.084232] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100240.085734] Call Trace: [100240.086286] dump_stack+0x41/0x60 [100240.086950] warn_alloc.cold.127+0x7b/0x108 [100240.087772] __alloc_pages_slowpath+0xcb2/0xcd0 [100240.088619] ? __alloc_pages_nodemask+0x166/0x330 [100240.089470] __alloc_pages_nodemask+0x2e2/0x330 [100240.090308] kmalloc_order+0x28/0x90 [100240.091006] kmalloc_order_trace+0x1d/0xb0 [100240.091784] __kmalloc+0x203/0x250 [100240.092487] virtqueue_add+0x493/0xc70 [100240.093208] virtqueue_add_sgs+0x80/0xa0 [100240.093953] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100240.094913] ? scsi_alloc_sgtables+0x84/0x1c0 [100240.095727] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100240.096621] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100240.097633] scsi_queue_rq+0x512/0xb10 [100240.098382] blk_mq_dispatch_rq_list+0x154/0x880 [100240.099246] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [100240.100078] __blk_mq_sched_dispatch_requests+0x110/0x140 [100240.101025] blk_mq_sched_dispatch_requests+0x30/0x60 [100240.101924] __blk_mq_run_hw_queue+0x2b/0x70 [100240.102735] __blk_mq_delay_run_hw_queue+0x141/0x160 [100240.103647] blk_mq_sched_insert_requests+0x6c/0xf0 [100240.104534] blk_mq_flush_plug_list+0x135/0x220 [100240.105366] blk_flush_plug_list+0xd7/0x100 [100240.106139] blk_finish_plug+0x25/0x36 [100240.106878] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100240.107928] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100240.108900] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.114424] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.115526] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100240.116478] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100240.117354] ofd_commitrw+0x5b4/0xd20 [ofd] [100240.118141] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.119125] obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.120129] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100240.121127] ? flush_work+0x42/0x1d0 [100240.121830] ? internal_add_timer+0x42/0x70 [100240.122611] ? _cond_resched+0x15/0x30 [100240.123332] ? mutex_lock+0xe/0x30 [100240.123999] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100240.125020] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100240.126124] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100240.127245] ? tgt_mask_cksum_types+0x70/0x70 [ptlrpc] [100240.128253] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100240.129144] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100240.130151] kthread+0x134/0x150 [100240.130815] ? set_kthread_struct+0x50/0x50 [100240.131603] ret_from_fork+0x1f/0x40 [100240.132319] CPU: 7 PID: 527068 Comm: ll_ost_io01_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100240.135345] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100240.137470] Call Trace: [100240.138263] dump_stack+0x41/0x60 [100240.139217] warn_alloc.cold.127+0x7b/0x108 [100240.140362] __alloc_pages_slowpath+0xcb2/0xcd0 [100240.141597] ? __alloc_pages_nodemask+0x166/0x330 [100240.142846] __alloc_pages_nodemask+0x2e2/0x330 [100240.144051] kmalloc_order+0x28/0x90 [100240.145079] kmalloc_order_trace+0x1d/0xb0 [100240.146186] __kmalloc+0x203/0x250 [100240.147158] virtqueue_add+0x493/0xc70 [100240.148182] ? finish_wait+0x80/0x80 [100240.149161] virtqueue_add_sgs+0x80/0xa0 [100240.150245] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100240.151587] ? scsi_alloc_sgtables+0x84/0x1c0 [100240.152773] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100240.154074] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100240.155495] scsi_queue_rq+0x512/0xb10 [100240.156534] __blk_mq_try_issue_directly+0x163/0x200 [100240.157820] blk_mq_request_issue_directly+0x4e/0xb0 [100240.159081] blk_mq_try_issue_list_directly+0x62/0x100 [100240.160375] blk_mq_sched_insert_requests+0xa4/0xf0 [100240.161612] blk_mq_flush_plug_list+0x135/0x220 [100240.162799] blk_flush_plug_list+0xd7/0x100 [100240.163927] blk_finish_plug+0x25/0x36 [100240.164977] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100240.166449] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100240.167803] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.169361] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.170929] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100240.172306] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100240.173574] ofd_commitrw+0x5b4/0xd20 [ofd] [100240.174690] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.176024] obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.177308] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100240.178620] ? work_busy+0x80/0x80 [100240.179565] ? internal_add_timer+0x42/0x70 [100240.180630] ? _cond_resched+0x15/0x30 [100240.181628] ? mutex_lock+0xe/0x30 [100240.182538] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100240.183890] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100240.185123] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100240.186598] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100240.187766] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100240.189192] kthread+0x134/0x150 [100240.190145] ? set_kthread_struct+0x50/0x50 [100240.191270] ret_from_fork+0x1f/0x40 [100240.192264] CPU: 9 PID: 527319 Comm: ll_ost_io02_055 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100240.195256] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100240.197105] Call Trace: [100240.197853] dump_stack+0x41/0x60 [100240.198768] warn_alloc.cold.127+0x7b/0x108 [100240.199862] __alloc_pages_slowpath+0xcb2/0xcd0 [100240.200885] ? __alloc_pages_nodemask+0x166/0x330 [100240.206861] __alloc_pages_nodemask+0x2e2/0x330 [100240.210724] kmalloc_order+0x28/0x90 [100240.211594] kmalloc_order_trace+0x1d/0xb0 [100240.212593] __kmalloc+0x203/0x250 [100240.213560] virtqueue_add+0x493/0xc70 [100240.214567] ? finish_wait+0x80/0x80 [100240.215445] virtqueue_add_sgs+0x80/0xa0 [100240.216485] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100240.217839] ? scsi_alloc_sgtables+0x84/0x1c0 [100240.219021] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100240.220318] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100240.221719] scsi_queue_rq+0x512/0xb10 [100240.222769] __blk_mq_try_issue_directly+0x163/0x200 [100240.224072] blk_mq_request_issue_directly+0x4e/0xb0 [100240.225189] blk_mq_try_issue_list_directly+0x62/0x100 [100240.226200] blk_mq_sched_insert_requests+0xa4/0xf0 [100240.227292] blk_mq_flush_plug_list+0x135/0x220 [100240.228287] blk_flush_plug_list+0xd7/0x100 [100240.229143] blk_finish_plug+0x25/0x36 [100240.230049] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100240.231357] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100240.232676] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.234232] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100240.235747] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100240.237069] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100240.238072] ofd_commitrw+0x5b4/0xd20 [ofd] [100240.239206] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.240566] obd_commitrw+0x1b6/0x370 [ptlrpc] [100240.241772] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100240.242988] ? flush_work+0x42/0x1d0 [100240.243769] ? internal_add_timer+0x42/0x70 [100240.244703] ? _cond_resched+0x15/0x30 [100240.245591] ? mutex_lock+0xe/0x30 [100240.246432] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100240.247646] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100240.249112] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100240.250516] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100240.251774] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100240.253045] kthread+0x134/0x150 [100240.253893] ? set_kthread_struct+0x50/0x50 [100240.254974] ret_from_fork+0x1f/0x40 [100245.833379] warn_alloc: 4 callbacks suppressed [100245.833382] ll_ost_io00_001: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100245.833391] ll_ost_io00_047: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100245.833395] ll_ost_io00_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100245.833404] CPU: 1 PID: 527755 Comm: ll_ost_io00_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100245.833408] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100245.833410] Call Trace: [100245.833416] dump_stack+0x41/0x60 [100245.833425] warn_alloc.cold.127+0x7b/0x108 [100245.833426] ll_ost_io04_004: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100245.833431] __alloc_pages_slowpath+0xcb2/0xcd0 [100245.833432] ,cpuset=/,mems_allowed=0 [100245.833435] ? __alloc_pages_nodemask+0x166/0x330 [100245.833439] __alloc_pages_nodemask+0x2e2/0x330 [100245.833442] kmalloc_order+0x28/0x90 [100245.833448] kmalloc_order_trace+0x1d/0xb0 [100245.833451] __kmalloc+0x203/0x250 [100245.833456] virtqueue_add+0x493/0xc70 [100245.833460] ll_ost_io04_066: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [100245.833462] ? finish_wait+0x80/0x80 [100245.833465] ,mems_allowed=0 [100245.833466] virtqueue_add_sgs+0x80/0xa0 [100245.833471] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100245.833476] ? scsi_alloc_sgtables+0x84/0x1c0 [100245.833481] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100245.833484] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100245.833488] scsi_queue_rq+0x512/0xb10 [100245.833491] __blk_mq_try_issue_directly+0x163/0x200 [100245.833497] blk_mq_request_issue_directly+0x4e/0xb0 [100245.833501] blk_mq_try_issue_list_directly+0x62/0x100 [100245.833504] blk_mq_sched_insert_requests+0xa4/0xf0 [100245.833508] blk_mq_flush_plug_list+0x135/0x220 [100245.833513] blk_flush_plug_list+0xd7/0x100 [100245.833518] blk_finish_plug+0x25/0x36 [100245.833536] ll_ost_io04_043: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [100245.833523] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100245.833544] ,mems_allowed=0 [100245.833558] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100245.833588] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.833614] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.833640] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100245.833667] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100245.833687] ofd_commitrw+0x5b4/0xd20 [ofd] [100245.833704] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.833832] obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.833939] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100245.834042] ? flush_work+0x42/0x1d0 [100245.834048] ? internal_add_timer+0x42/0x70 [100245.834051] ? _cond_resched+0x15/0x30 [100245.834054] ? mutex_lock+0xe/0x30 [100245.834057] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100245.834160] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100245.834264] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100245.834366] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100245.834492] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100245.834600] kthread+0x134/0x150 [100245.834604] ? set_kthread_struct+0x50/0x50 [100245.834606] ret_from_fork+0x1f/0x40 [100245.834611] warn_alloc_show_mem: 9 callbacks suppressed [100245.834613] Mem-Info: [100245.834612] CPU: 18 PID: 527163 Comm: ll_ost_io04_004 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100245.834616] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100245.834615] active_anon:27900 inactive_anon:167682 isolated_anon:0 active_file:27487171 inactive_file:5309521 isolated_file:11 unevictable:0 dirty:2850 writeback:5 slab_reclaimable:1027091 slab_unreclaimable:616218 mapped:22307 shmem:145 pagetables:5800 bounce:0 free:3378695 free_pcp:510 free_cma:0 [100245.834617] Call Trace: [100245.834621] Node 0 active_anon:111600kB inactive_anon:670728kB active_file:109948684kB inactive_file:21238084kB unevictable:0kB isolated(anon):0kB isolated(file):44kB mapped:89228kB dirty:11400kB writeback:20kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35632kB pagetables:23200kB all_unreclaimable? no [100245.834621] dump_stack+0x41/0x60 [100245.834627] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100245.834628] warn_alloc.cold.127+0x7b/0x108 [100245.834633] lowmem_reserve[]: 0 913 [100245.834633] __alloc_pages_slowpath+0xcb2/0xcd0 [100245.834635] 150007 150007 150007 [100245.834638] Node 0 [100245.834637] ? __alloc_pages_nodemask+0x166/0x330 [100245.834638] DMA32 free:596512kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:4kB local_pcp:0kB free_cma:0kB [100245.834640] __alloc_pages_nodemask+0x2e2/0x330 [100245.834643] lowmem_reserve[]: 0 0 149093 149093 [100245.834643] kmalloc_order+0x28/0x90 [100245.834646] 149093 [100245.834648] Node 0 [100245.834648] kmalloc_order_trace+0x1d/0xb0 [100245.834648] Normal free:12907004kB min:49268kB low:201940kB high:354612kB active_anon:111220kB inactive_anon:670724kB active_file:109948420kB inactive_file:21238300kB unevictable:0kB writepending:11420kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2052kB local_pcp:16kB free_cma:0kB [100245.834654] lowmem_reserve[]: 0 [100245.834652] __kmalloc+0x203/0x250 [100245.834655] 0 0 0 0 [100245.834658] Node 0 DMA: [100245.834657] ? __wake_up_common+0x7a/0x190 [100245.834659] 0*4kB 0*8kB 0*16kB 0*32kB [100245.834660] virtqueue_add+0x493/0xc70 [100245.834662] 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) [100245.834666] virtqueue_add_sgs+0x80/0xa0 [100245.834667] 1*2048kB (M) 2*4096kB (M) = 11264kB [100245.834671] Node 0 DMA32: 18*4kB [100245.834671] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100245.834674] (UM) 21*8kB (UMH) 47*16kB (UME) 40*32kB (UMEH) 39*64kB [100245.834676] ? scsi_alloc_sgtables+0x84/0x1c0 [100245.834679] (UM) 41*128kB (UMH) 43*256kB (UMH) [100245.834681] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100245.834682] 44*512kB (UMEH) 38*1024kB (UMH) 41*2048kB [100245.834685] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100245.834686] (UM) 105*4096kB (UM) = 596512kB [100245.834689] Node 0 [100245.834688] scsi_queue_rq+0x512/0xb10 [100245.834689] Normal: 1646092*4kB (UME) 727191*8kB (UME) 31529*16kB (UEH) [100245.834691] __blk_mq_try_issue_directly+0x163/0x200 [100245.834695] 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB [100245.834697] blk_mq_request_issue_directly+0x4e/0xb0 [100245.834700] 0*2048kB 0*4096kB = 12906360kB [100245.834702] blk_mq_try_issue_list_directly+0x62/0x100 [100245.834703] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100245.834705] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100245.834707] 32796633 total pagecache pages [100245.834705] blk_mq_sched_insert_requests+0xa4/0xf0 [100245.834708] 1327 pages in swap cache [100245.834709] Swap cache stats: add 1720006, delete 1718698, find 98361/118953 [100245.834710] Free swap = 5008828kB [100245.834710] blk_mq_flush_plug_list+0x135/0x220 [100245.834711] Total swap = 11075580kB [100245.834712] 39321462 pages RAM [100245.834713] 0 pages HighMem/MovableOnly [100245.834714] 905848 pages reserved [100245.834714] 0 pages hwpoisoned [100245.834714] blk_flush_plug_list+0xd7/0x100 [100245.834719] blk_finish_plug+0x25/0x36 [100245.834724] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100245.834755] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100245.834782] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.834807] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.834833] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100245.834848] ,cpuset= [100245.834858] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100245.834876] ofd_commitrw+0x5b4/0xd20 [ofd] [100245.834893] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.835007] obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.835115] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100245.835220] ? flush_work+0x42/0x1d0 [100245.835224] ? internal_add_timer+0x42/0x70 [100245.835227] ? _cond_resched+0x15/0x30 [100245.835229] ? mutex_lock+0xe/0x30 [100245.835232] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100245.835336] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100245.835449] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100245.835555] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100245.835666] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100245.835781] kthread+0x134/0x150 [100245.835784] ? set_kthread_struct+0x50/0x50 [100245.835787] ret_from_fork+0x1f/0x40 [100245.835793] CPU: 19 PID: 527999 Comm: ll_ost_io04_066 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100245.835796] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100245.835797] Call Trace: [100245.835799] dump_stack+0x41/0x60 [100245.835804] warn_alloc.cold.127+0x7b/0x108 [100245.835809] __alloc_pages_slowpath+0xcb2/0xcd0 [100245.835813] __alloc_pages_nodemask+0x2e2/0x330 [100245.835816] kmalloc_order+0x28/0x90 [100245.835820] kmalloc_order_trace+0x1d/0xb0 [100245.835824] __kmalloc+0x203/0x250 [100245.835828] virtqueue_add+0x493/0xc70 [100245.835834] ? finish_wait+0x80/0x80 [100245.835837] virtqueue_add_sgs+0x80/0xa0 [100245.835842] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100245.835847] ? scsi_alloc_sgtables+0x84/0x1c0 [100245.835852] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100245.835856] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100245.835859] scsi_queue_rq+0x512/0xb10 [100245.835862] __blk_mq_try_issue_directly+0x163/0x200 [100245.835868] blk_mq_request_issue_directly+0x4e/0xb0 [100245.835872] blk_mq_try_issue_list_directly+0x62/0x100 [100245.835875] blk_mq_sched_insert_requests+0xa4/0xf0 [100245.835879] blk_mq_flush_plug_list+0x135/0x220 [100245.835884] blk_flush_plug_list+0xd7/0x100 [100245.835889] blk_finish_plug+0x25/0x36 [100245.835894] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100245.835928] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100245.835956] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.835984] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.836011] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100245.836038] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100245.836061] ofd_commitrw+0x5b4/0xd20 [ofd] [100245.836079] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.836205] obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.836316] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100245.836432] ? flush_work+0x42/0x1d0 [100245.836437] ? internal_add_timer+0x42/0x70 [100245.836441] ? _cond_resched+0x15/0x30 [100245.836443] ? mutex_lock+0xe/0x30 [100245.836447] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100245.836568] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100245.836690] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100245.836801] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100245.836911] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100245.837035] kthread+0x134/0x150 [100245.837040] ? set_kthread_struct+0x50/0x50 [100245.837043] ret_from_fork+0x1f/0x40 [100245.837050] CPU: 16 PID: 527680 Comm: ll_ost_io04_043 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100245.837055] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100245.837057] Call Trace: [100245.837061] dump_stack+0x41/0x60 [100245.837067] warn_alloc.cold.127+0x7b/0x108 [100245.837073] __alloc_pages_slowpath+0xcb2/0xcd0 [100245.837077] ? __alloc_pages_nodemask+0x166/0x330 [100245.837081] __alloc_pages_nodemask+0x2e2/0x330 [100245.837084] kmalloc_order+0x28/0x90 [100245.837089] kmalloc_order_trace+0x1d/0xb0 [100245.837093] __kmalloc+0x203/0x250 [100245.837098] virtqueue_add+0x493/0xc70 [100245.837103] ? finish_wait+0x80/0x80 [100245.837107] virtqueue_add_sgs+0x80/0xa0 [100245.837112] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100245.837117] ? scsi_alloc_sgtables+0x84/0x1c0 [100245.837123] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100245.837126] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100245.837130] scsi_queue_rq+0x512/0xb10 [100245.837132] __blk_mq_try_issue_directly+0x163/0x200 [100245.837138] blk_mq_request_issue_directly+0x4e/0xb0 [100245.837142] blk_mq_try_issue_list_directly+0x62/0x100 [100245.837146] blk_mq_sched_insert_requests+0xa4/0xf0 [100245.837151] blk_mq_flush_plug_list+0x135/0x220 [100245.837155] blk_flush_plug_list+0xd7/0x100 [100245.837160] blk_finish_plug+0x25/0x36 [100245.837165] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100245.837201] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100245.837233] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.837261] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.837291] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100245.837319] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100245.837341] ofd_commitrw+0x5b4/0xd20 [ofd] [100245.837359] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.837480] obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.837595] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100245.837708] ? flush_work+0x42/0x1d0 [100245.837714] ? internal_add_timer+0x42/0x70 [100245.837717] ? _cond_resched+0x15/0x30 [100245.837720] ? mutex_lock+0xe/0x30 [100245.837724] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100245.837848] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100245.837972] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100245.838061] ,cpuset= [100245.838079] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100245.838212] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100245.838328] kthread+0x134/0x150 [100245.838333] ? set_kthread_struct+0x50/0x50 [100245.838335] ret_from_fork+0x1f/0x40 [100245.839630] ll_ost_io04_033: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100245.839641] CPU: 17 PID: 527279 Comm: ll_ost_io04_033 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100245.839645] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100245.839646] Call Trace: [100245.839649] dump_stack+0x41/0x60 [100245.839656] warn_alloc.cold.127+0x7b/0x108 [100245.839662] __alloc_pages_slowpath+0xcb2/0xcd0 [100245.839667] ? __alloc_pages_nodemask+0x166/0x330 [100245.839670] __alloc_pages_nodemask+0x2e2/0x330 [100245.839674] kmalloc_order+0x28/0x90 [100245.839679] kmalloc_order_trace+0x1d/0xb0 [100245.839683] __kmalloc+0x203/0x250 [100245.839688] ? __wake_up_common+0x7a/0x190 [100245.839692] virtqueue_add+0x493/0xc70 [100245.839699] virtqueue_add_sgs+0x80/0xa0 [100245.839704] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100245.839709] ? scsi_alloc_sgtables+0x84/0x1c0 [100245.839716] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100245.839720] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100245.839724] scsi_queue_rq+0x512/0xb10 [100245.839727] __blk_mq_try_issue_directly+0x163/0x200 [100245.839733] blk_mq_request_issue_directly+0x4e/0xb0 [100245.839737] blk_mq_try_issue_list_directly+0x62/0x100 [100245.839742] blk_mq_sched_insert_requests+0xa4/0xf0 [100245.839746] blk_mq_flush_plug_list+0x135/0x220 [100245.839751] blk_flush_plug_list+0xd7/0x100 [100245.839757] blk_finish_plug+0x25/0x36 [100245.839762] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100245.839799] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100245.839833] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.839862] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.839891] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100245.839921] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100245.839942] ofd_commitrw+0x5b4/0xd20 [ofd] [100245.839961] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.840080] obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.840206] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100245.840330] ? flush_work+0x42/0x1d0 [100245.840337] ? internal_add_timer+0x42/0x70 [100245.840341] ? _cond_resched+0x15/0x30 [100245.840343] ? mutex_lock+0xe/0x30 [100245.840347] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100245.840474] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100245.840581] / [100245.840600] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100245.840723] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100245.840853] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100245.840975] kthread+0x134/0x150 [100245.840979] ? set_kthread_struct+0x50/0x50 [100245.840983] ret_from_fork+0x1f/0x40 [100245.860356] ll_ost_io04_083: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100245.860817] ,mems_allowed=0 [100245.860818] / [100245.860822] CPU: 2 PID: 188098 Comm: ll_ost_io00_001 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100245.860826] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100245.860828] Call Trace: [100245.860832] dump_stack+0x41/0x60 [100245.860839] warn_alloc.cold.127+0x7b/0x108 [100245.860845] __alloc_pages_slowpath+0xcb2/0xcd0 [100245.860848] ? __alloc_pages_nodemask+0x166/0x330 [100245.860852] __alloc_pages_nodemask+0x2e2/0x330 [100245.860855] kmalloc_order+0x28/0x90 [100245.860860] kmalloc_order_trace+0x1d/0xb0 [100245.860864] __kmalloc+0x203/0x250 [100245.860868] virtqueue_add+0x493/0xc70 [100245.860874] ? finish_wait+0x80/0x80 [100245.860878] virtqueue_add_sgs+0x80/0xa0 [100245.860883] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100245.860889] ? scsi_alloc_sgtables+0x84/0x1c0 [100245.860894] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100245.860898] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100245.860901] scsi_queue_rq+0x512/0xb10 [100245.860904] __blk_mq_try_issue_directly+0x163/0x200 [100245.860911] blk_mq_request_issue_directly+0x4e/0xb0 [100245.860915] blk_mq_try_issue_list_directly+0x62/0x100 [100245.860919] blk_mq_sched_insert_requests+0xa4/0xf0 [100245.860924] blk_mq_flush_plug_list+0x135/0x220 [100245.860928] blk_flush_plug_list+0xd7/0x100 [100245.860933] blk_finish_plug+0x25/0x36 [100245.860939] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100245.860973] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100245.861003] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.861029] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.861056] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100245.861081] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100245.861102] ofd_commitrw+0x5b4/0xd20 [ofd] [100245.861118] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.861237] obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.861343] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100245.861448] ? flush_work+0x42/0x1d0 [100245.861453] ? internal_add_timer+0x42/0x70 [100245.861457] ? _cond_resched+0x15/0x30 [100245.861459] ? mutex_lock+0xe/0x30 [100245.861463] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100245.861576] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100245.861673] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100245.861735] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100245.861804] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100245.861871] kthread+0x134/0x150 [100245.861874] ? set_kthread_struct+0x50/0x50 [100245.861877] ret_from_fork+0x1f/0x40 [100245.861907] ,cpuset= [100245.862042] ll_ost_io04_038: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/ [100245.862984] ,mems_allowed=0 [100245.862989] CPU: 0 PID: 527424 Comm: ll_ost_io00_047 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100245.862993] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100245.862994] Call Trace: [100245.862997] dump_stack+0x41/0x60 [100245.863003] warn_alloc.cold.127+0x7b/0x108 [100245.863014] __alloc_pages_slowpath+0xcb2/0xcd0 [100245.863022] ? __alloc_pages_nodemask+0x166/0x330 [100245.863026] __alloc_pages_nodemask+0x2e2/0x330 [100245.863029] kmalloc_order+0x28/0x90 [100245.863034] kmalloc_order_trace+0x1d/0xb0 [100245.863037] __kmalloc+0x203/0x250 [100245.863046] virtqueue_add+0x493/0xc70 [100245.863052] ? finish_wait+0x80/0x80 [100245.863056] virtqueue_add_sgs+0x80/0xa0 [100245.863065] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100245.863070] ? scsi_alloc_sgtables+0x84/0x1c0 [100245.863076] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100245.863085] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100245.863088] scsi_queue_rq+0x512/0xb10 [100245.863092] __blk_mq_try_issue_directly+0x163/0x200 [100245.863102] blk_mq_request_issue_directly+0x4e/0xb0 [100245.863107] blk_mq_try_issue_list_directly+0x62/0x100 [100245.863115] blk_mq_sched_insert_requests+0xa4/0xf0 [100245.863119] blk_mq_flush_plug_list+0x135/0x220 [100245.863123] blk_flush_plug_list+0xd7/0x100 [100245.863133] blk_finish_plug+0x25/0x36 [100245.863138] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100245.863186] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.863222] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100245.863265] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100245.863306] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100245.863340] ofd_commitrw+0x5b4/0xd20 [ofd] [100245.863363] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.863536] obd_commitrw+0x1b6/0x370 [ptlrpc] [100245.863687] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100245.863848] ? flush_work+0x42/0x1d0 [100245.863853] ? internal_add_timer+0x42/0x70 [100245.863856] ? _cond_resched+0x15/0x30 [100245.863859] ? mutex_lock+0xe/0x30 [100245.863868] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100245.863998] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100245.864157] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100245.864309] ? lu_context_exit+0x41/0x70 [obdclass] [100245.864412] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100245.864570] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100245.864723] kthread+0x134/0x150 [100245.864728] ? set_kthread_struct+0x50/0x50 [100245.864736] ret_from_fork+0x1f/0x40 [100245.865325] ll_ost_io04_072: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [100246.327647] ,mems_allowed=0 [100246.327648] / [100246.327652] CPU: 18 PID: 527566 Comm: ll_ost_io04_038 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100246.329811] / [100246.329810] ,mems_allowed=0 [100246.329813] ,mems_allowed=0 [100246.330461] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100246.331465] Call Trace: [100246.338050] dump_stack+0x41/0x60 [100246.338759] warn_alloc.cold.127+0x7b/0x108 [100246.339557] __alloc_pages_slowpath+0xcb2/0xcd0 [100246.340397] ? __alloc_pages_nodemask+0x166/0x330 [100246.341262] __alloc_pages_nodemask+0x2e2/0x330 [100246.342103] kmalloc_order+0x28/0x90 [100246.342814] kmalloc_order_trace+0x1d/0xb0 [100246.343593] __kmalloc+0x203/0x250 [100246.344276] virtqueue_add+0x493/0xc70 [100246.345001] ? finish_wait+0x80/0x80 [100246.345705] virtqueue_add_sgs+0x80/0xa0 [100246.346464] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100246.347505] ? scsi_alloc_sgtables+0x84/0x1c0 [100246.348330] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100246.349278] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100246.350278] scsi_queue_rq+0x512/0xb10 [100246.351010] __blk_mq_try_issue_directly+0x163/0x200 [100246.351916] blk_mq_request_issue_directly+0x4e/0xb0 [100246.352832] blk_mq_try_issue_list_directly+0x62/0x100 [100246.353762] blk_mq_sched_insert_requests+0xa4/0xf0 [100246.354654] blk_mq_flush_plug_list+0x135/0x220 [100246.355483] blk_flush_plug_list+0xd7/0x100 [100246.356264] blk_finish_plug+0x25/0x36 [100246.356994] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100246.358044] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100246.359224] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100246.360326] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100246.361271] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100246.362179] ofd_commitrw+0x5b4/0xd20 [ofd] [100246.362998] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100246.363957] obd_commitrw+0x1b6/0x370 [ptlrpc] [100246.364880] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100246.365820] ? flush_work+0x42/0x1d0 [100246.366532] ? internal_add_timer+0x42/0x70 [100246.367427] ? _cond_resched+0x15/0x30 [100246.368258] ? mutex_lock+0xe/0x30 [100246.369250] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100246.370685] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100246.372035] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100246.373420] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100246.374712] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100246.376132] kthread+0x134/0x150 [100246.376970] ? set_kthread_struct+0x50/0x50 [100246.378063] ret_from_fork+0x1f/0x40 [100246.379037] CPU: 17 PID: 528060 Comm: ll_ost_io04_072 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100246.381436] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100246.383306] Call Trace: [100246.383930] dump_stack+0x41/0x60 [100246.384692] warn_alloc.cold.127+0x7b/0x108 [100246.385590] __alloc_pages_slowpath+0xcb2/0xcd0 [100246.386542] ? __alloc_pages_nodemask+0x166/0x330 [100246.387527] __alloc_pages_nodemask+0x2e2/0x330 [100246.388519] kmalloc_order+0x28/0x90 [100246.389348] kmalloc_order_trace+0x1d/0xb0 [100246.390244] __kmalloc+0x203/0x250 [100246.391026] virtqueue_add+0x493/0xc70 [100246.391869] ? finish_wait+0x80/0x80 [100246.392676] virtqueue_add_sgs+0x80/0xa0 [100246.393542] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100246.394659] ? scsi_alloc_sgtables+0x84/0x1c0 [100246.395608] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100246.396652] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100246.397812] scsi_queue_rq+0x512/0xb10 [100246.398670] __blk_mq_try_issue_directly+0x163/0x200 [100246.399712] blk_mq_request_issue_directly+0x4e/0xb0 [100246.400764] blk_mq_try_issue_list_directly+0x62/0x100 [100246.401852] blk_mq_sched_insert_requests+0xa4/0xf0 [100246.402902] blk_mq_flush_plug_list+0x135/0x220 [100246.403874] blk_flush_plug_list+0xd7/0x100 [100246.404799] blk_finish_plug+0x25/0x36 [100246.405632] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100246.406823] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100246.408119] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100246.409374] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100246.410479] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100246.411517] ofd_commitrw+0x5b4/0xd20 [ofd] [100246.412446] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100246.413545] obd_commitrw+0x1b6/0x370 [ptlrpc] [100246.414612] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100246.415716] ? flush_work+0x42/0x1d0 [100246.416536] ? internal_add_timer+0x42/0x70 [100246.417453] ? _cond_resched+0x15/0x30 [100246.418318] ? mutex_lock+0xe/0x30 [100246.419105] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100246.420265] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100246.421529] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100246.422800] ? lu_context_exit+0x2f/0x70 [obdclass] [100246.423890] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100246.424952] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100246.426122] kthread+0x134/0x150 [100246.426884] ? set_kthread_struct+0x50/0x50 [100246.427779] ret_from_fork+0x1f/0x40 [100246.428621] CPU: 16 PID: 528071 Comm: ll_ost_io04_083 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100246.431627] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100246.433704] Call Trace: [100246.434518] dump_stack+0x41/0x60 [100246.435462] warn_alloc.cold.127+0x7b/0x108 [100246.436610] __alloc_pages_slowpath+0xcb2/0xcd0 [100246.437855] ? __alloc_pages_nodemask+0x166/0x330 [100246.439145] __alloc_pages_nodemask+0x2e2/0x330 [100246.440298] kmalloc_order+0x28/0x90 [100246.441299] kmalloc_order_trace+0x1d/0xb0 [100246.442416] __kmalloc+0x203/0x250 [100246.443403] virtqueue_add+0x493/0xc70 [100246.444457] ? finish_wait+0x80/0x80 [100246.445486] virtqueue_add_sgs+0x80/0xa0 [100246.446545] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100246.447917] ? scsi_alloc_sgtables+0x84/0x1c0 [100246.449039] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100246.450329] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100246.451741] scsi_queue_rq+0x512/0xb10 [100246.452770] __blk_mq_try_issue_directly+0x163/0x200 [100246.454062] blk_mq_request_issue_directly+0x4e/0xb0 [100246.455379] blk_mq_try_issue_list_directly+0x62/0x100 [100246.456688] blk_mq_sched_insert_requests+0xa4/0xf0 [100246.457959] blk_mq_flush_plug_list+0x135/0x220 [100246.459201] blk_flush_plug_list+0xd7/0x100 [100246.460334] blk_finish_plug+0x25/0x36 [100246.461403] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100246.462908] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100246.464278] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100246.465861] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100246.467395] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100246.468732] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100246.469993] ofd_commitrw+0x5b4/0xd20 [ofd] [100246.471133] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100246.472521] obd_commitrw+0x1b6/0x370 [ptlrpc] [100246.473851] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100246.475247] ? flush_work+0x42/0x1d0 [100246.476256] ? internal_add_timer+0x42/0x70 [100246.477405] ? _cond_resched+0x15/0x30 [100246.478451] ? mutex_lock+0xe/0x30 [100246.479416] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100246.480843] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100246.482393] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100246.483949] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100246.485220] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100246.486618] kthread+0x134/0x150 [100246.487575] ? set_kthread_struct+0x50/0x50 [100246.488705] ret_from_fork+0x1f/0x40 [100787.142109] warn_alloc: 36 callbacks suppressed [100787.142113] ll_ost_io00_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.146497] CPU: 0 PID: 527174 Comm: ll_ost_io00_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.148628] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.150091] Call Trace: [100787.150662] dump_stack+0x41/0x60 [100787.151337] warn_alloc.cold.127+0x7b/0x108 [100787.152102] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.152937] ? __alloc_pages_nodemask+0x166/0x330 [100787.153786] __alloc_pages_nodemask+0x2e2/0x330 [100787.154636] ll_ost_io00_034: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.154688] kmalloc_order+0x28/0x90 [100787.155264] ll_ost_io02_034: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.159571] kmalloc_order_trace+0x1d/0xb0 [100787.160362] __kmalloc+0x203/0x250 [100787.161087] virtqueue_add+0x493/0xc70 [100787.161820] virtqueue_add_sgs+0x80/0xa0 [100787.162588] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.163588] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.164406] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.165367] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.166382] scsi_queue_rq+0x512/0xb10 [100787.167112] __blk_mq_try_issue_directly+0x163/0x200 [100787.168001] blk_mq_request_issue_directly+0x4e/0xb0 [100787.168888] blk_mq_try_issue_list_directly+0x62/0x100 [100787.169803] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.170669] blk_mq_flush_plug_list+0x135/0x220 [100787.171529] blk_flush_plug_list+0xd7/0x100 [100787.172323] blk_finish_plug+0x25/0x36 [100787.173046] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.174137] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.175248] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.176345] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.176568] ll_ost_io01_060: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.177323] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.179263] ll_ost_io02_072: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.179273] ll_ost_io04_032: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.179330] ll_ost_io02_045: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.182231] ll_ost_io02_075: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100787.182479] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.184886] ,cpuset= [100787.186886] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.188867] / [100787.190098] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.190752] ,mems_allowed=0 [100787.192011] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.193754] ? flush_work+0x42/0x1d0 [100787.197356] ? internal_add_timer+0x42/0x70 [100787.198490] ? _cond_resched+0x15/0x30 [100787.199535] ? mutex_lock+0xe/0x30 [100787.200485] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.201784] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.202845] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.203943] ? finish_wait+0x80/0x80 [100787.204667] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.205563] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.206531] kthread+0x134/0x150 [100787.207164] ? set_kthread_struct+0x50/0x50 [100787.207927] ret_from_fork+0x1f/0x40 [100787.208609] warn_alloc_show_mem: 9 callbacks suppressed [100787.208609] CPU: 11 PID: 527863 Comm: ll_ost_io02_072 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.208616] Mem-Info: [100787.209760] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.211904] active_anon:27565 inactive_anon:151798 isolated_anon:0 active_file:27897165 inactive_file:6849977 isolated_file:12 unevictable:0 dirty:8020 writeback:30 slab_reclaimable:1019160 slab_unreclaimable:653031 mapped:22174 shmem:145 pagetables:5818 bounce:0 free:1411725 free_pcp:368 free_cma:0 [100787.212553] Call Trace: [100787.214038] Node 0 active_anon:110260kB inactive_anon:607192kB active_file:111588660kB inactive_file:27399908kB unevictable:0kB isolated(anon):0kB isolated(file):48kB mapped:88696kB dirty:32080kB writeback:120kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35568kB pagetables:23272kB all_unreclaimable? no [100787.220824] dump_stack+0x41/0x60 [100787.221434] Node 0 [100787.227158] warn_alloc.cold.127+0x7b/0x108 [100787.227873] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100787.228516] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.229364] lowmem_reserve[]: [100787.233435] ? __alloc_pages_nodemask+0x166/0x330 [100787.234343] 0 [100787.235159] __alloc_pages_nodemask+0x2e2/0x330 [100787.236067] 913 [100787.236340] ll_ost_io00_078: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.236368] ll_ost_io00_057: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100787.236646] kmalloc_order+0x28/0x90 [100787.237683] 150007 [100787.238290] kmalloc_order_trace+0x1d/0xb0 [100787.240571] 150007 [100787.242908] __kmalloc+0x203/0x250 [100787.243795] 150007 [100787.244471] ? common_interrupt+0xa/0xf [100787.246054] virtqueue_add+0x493/0xc70 [100787.246923] Node 0 [100787.247594] ? finish_wait+0x80/0x80 [100787.248521] DMA32 free:596472kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100787.249084] virtqueue_add_sgs+0x80/0xa0 [100787.249977] lowmem_reserve[]: [100787.250634] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.251503] 0 [100787.255821] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.256780] 0 [100787.257600] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.258788] 149093 [100787.259413] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.260438] 149093 [100787.261049] scsi_queue_rq+0x512/0xb10 [100787.262176] 149093 [100787.262853] __blk_mq_try_issue_directly+0x163/0x200 [100787.264730] blk_mq_request_issue_directly+0x4e/0xb0 [100787.265636] Node 0 [100787.266290] blk_mq_try_issue_list_directly+0x62/0x100 [100787.267371] Normal free:5051700kB min:49268kB low:201940kB high:354612kB active_anon:109880kB inactive_anon:607188kB active_file:111589664kB inactive_file:27386932kB unevictable:0kB writepending:31232kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1556kB local_pcp:116kB free_cma:0kB [100787.267921] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.269028] lowmem_reserve[]: [100787.269698] blk_mq_flush_plug_list+0x135/0x220 [100787.270822] 0 [100787.275762] blk_flush_plug_list+0xd7/0x100 [100787.276840] 0 [100787.277664] blk_finish_plug+0x25/0x36 [100787.278696] 0 [100787.279291] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.280245] 0 [100787.280826] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.281708] 0 [100787.282304] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.284070] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.285171] Node 0 [100787.285744] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.287044] DMA: [100787.287624] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.288915] 0*4kB [100787.289579] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.290719] 0*8kB [100787.291354] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.292386] 0*16kB [100787.293032] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.293963] 0*32kB [100787.294605] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.295596] 0*64kB [100787.296258] ? flush_work+0x42/0x1d0 [100787.297236] 0*128kB [100787.297882] ? internal_add_timer+0x42/0x70 [100787.298939] 0*256kB [100787.299606] ? _cond_resched+0x15/0x30 [100787.300481] 0*512kB [100787.301143] ? mutex_lock+0xe/0x30 [100787.302105] 1*1024kB [100787.302777] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.303655] (U) [100787.304328] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.305146] 1*2048kB [100787.305829] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.306919] (M) [100787.307557] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.308750] 2*4096kB [100787.309441] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.310628] (M) [100787.311235] kthread+0x134/0x150 [100787.312223] = 11264kB [100787.312904] ? set_kthread_struct+0x50/0x50 [100787.313995] Node 0 [100787.314622] ret_from_fork+0x1f/0x40 [100787.315436] DMA32: [100787.316122] CPU: 3 PID: 527783 Comm: ll_ost_io00_078 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.317023] 16*4kB [100787.317796] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.318684] (UM) [100787.319352] Call Trace: [100787.321857] 19*8kB [100787.322525] dump_stack+0x41/0x60 [100787.324428] (UMH) [100787.325073] warn_alloc.cold.127+0x7b/0x108 [100787.326155] 48*16kB [100787.326788] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.327606] (UME) [100787.328240] ? __alloc_pages_nodemask+0x166/0x330 [100787.329170] 40*32kB [100787.329814] __alloc_pages_nodemask+0x2e2/0x330 [100787.330797] (UMEH) [100787.331413] kmalloc_order+0x28/0x90 [100787.332424] 37*64kB [100787.333059] kmalloc_order_trace+0x1d/0xb0 [100787.334045] (UM) [100787.334667] __kmalloc+0x203/0x250 [100787.335503] 38*128kB [100787.336126] virtqueue_add+0x493/0xc70 [100787.337041] (UM) [100787.337621] ? finish_wait+0x80/0x80 [100787.338424] 40*256kB [100787.339066] virtqueue_add_sgs+0x80/0xa0 [100787.339932] (UM) [100787.340528] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.341365] 42*512kB [100787.342025] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.342914] (UMEH) [100787.343510] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.344644] 36*1024kB [100787.345306] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.346258] (UMH) [100787.346884] scsi_queue_rq+0x512/0xb10 [100787.347955] 39*2048kB [100787.348630] __blk_mq_try_issue_directly+0x163/0x200 [100787.349796] (UM) [100787.350422] blk_mq_request_issue_directly+0x4e/0xb0 [100787.351294] 107*4096kB [100787.351963] blk_mq_try_issue_list_directly+0x62/0x100 [100787.353017] (UM) [100787.353626] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.354689] = 596248kB [100787.355372] blk_mq_flush_plug_list+0x135/0x220 [100787.356467] Node 0 [100787.357073] blk_flush_plug_list+0xd7/0x100 [100787.358130] Normal: [100787.358800] blk_finish_plug+0x25/0x36 [100787.359782] 371362*4kB [100787.360413] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.361339] (UME) [100787.361977] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.362823] 397547*8kB [100787.363489] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.364660] (UME) [100787.365263] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.366342] 24749*16kB [100787.367008] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.368255] (UMEH) [100787.368851] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.370075] 0*32kB [100787.370792] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.371903] 0*64kB [100787.372538] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.373526] 0*128kB [100787.374155] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.375049] 0*256kB [100787.375676] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.376647] 0*512kB [100787.377284] ? flush_work+0x42/0x1d0 [100787.378248] 0*1024kB [100787.378876] ? internal_add_timer+0x42/0x70 [100787.379874] 0*2048kB [100787.380496] ? _cond_resched+0x15/0x30 [100787.381334] 0*4096kB [100787.381982] ? mutex_lock+0xe/0x30 [100787.382905] = 5061808kB [100787.383560] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.384389] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100787.385045] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.385824] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100787.386496] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.387524] 34726005 total pagecache pages [100787.389226] ? finish_wait+0x80/0x80 [100787.390436] 1311 pages in swap cache [100787.392095] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.393275] Swap cache stats: add 1750730, delete 1749453, find 101026/122176 [100787.394225] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.395063] Free swap = 4907784kB [100787.395951] kthread+0x134/0x150 [100787.396954] Total swap = 11075580kB [100787.398416] ? set_kthread_struct+0x50/0x50 [100787.399512] 39321462 pages RAM [100787.400381] ret_from_fork+0x1f/0x40 [100787.401194] 0 pages HighMem/MovableOnly [100787.402072] CPU: 16 PID: 527278 Comm: ll_ost_io04_032 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.403137] 905848 pages reserved [100787.403969] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.404851] 0 pages hwpoisoned [100787.405685] Call Trace: [100787.411592] dump_stack+0x41/0x60 [100787.412291] warn_alloc.cold.127+0x7b/0x108 [100787.413085] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.413900] ? __alloc_pages_nodemask+0x166/0x330 [100787.414787] __alloc_pages_nodemask+0x2e2/0x330 [100787.415640] kmalloc_order+0x28/0x90 [100787.416344] kmalloc_order_trace+0x1d/0xb0 [100787.417164] __kmalloc+0x203/0x250 [100787.418101] virtqueue_add+0x493/0xc70 [100787.419087] ? finish_wait+0x80/0x80 [100787.420091] virtqueue_add_sgs+0x80/0xa0 [100787.421154] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.422490] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.423614] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.424866] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.426202] scsi_queue_rq+0x512/0xb10 [100787.427133] __blk_mq_try_issue_directly+0x163/0x200 [100787.428334] blk_mq_request_issue_directly+0x4e/0xb0 [100787.429531] blk_mq_try_issue_list_directly+0x62/0x100 [100787.430779] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.431664] blk_mq_flush_plug_list+0x135/0x220 [100787.432696] blk_flush_plug_list+0xd7/0x100 [100787.433732] blk_finish_plug+0x25/0x36 [100787.434710] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.436176] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.437605] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.439145] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.440664] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.442031] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.443246] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.444397] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.445752] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.446999] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.448303] ? flush_work+0x42/0x1d0 [100787.449318] ? internal_add_timer+0x42/0x70 [100787.450429] ? _cond_resched+0x15/0x30 [100787.451469] ? mutex_lock+0xe/0x30 [100787.452327] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.453652] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.455181] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.456463] ? finish_wait+0x80/0x80 [100787.457146] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.458386] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.459850] kthread+0x134/0x150 [100787.460642] ? set_kthread_struct+0x50/0x50 [100787.461460] ret_from_fork+0x1f/0x40 [100787.462143] CPU: 9 PID: 527866 Comm: ll_ost_io02_075 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.464549] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.466281] Call Trace: [100787.466909] dump_stack+0x41/0x60 [100787.467685] warn_alloc.cold.127+0x7b/0x108 [100787.468592] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.469569] __alloc_pages_nodemask+0x2e2/0x330 [100787.470530] kmalloc_order+0x28/0x90 [100787.471341] kmalloc_order_trace+0x1d/0xb0 [100787.472232] __kmalloc+0x203/0x250 [100787.473013] virtqueue_add+0x493/0xc70 [100787.473864] ? finish_wait+0x80/0x80 [100787.474677] virtqueue_add_sgs+0x80/0xa0 [100787.475541] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.476626] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.477559] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.478588] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.479715] scsi_queue_rq+0x512/0xb10 [100787.480569] __blk_mq_try_issue_directly+0x163/0x200 [100787.481624] blk_mq_request_issue_directly+0x4e/0xb0 [100787.482663] blk_mq_try_issue_list_directly+0x62/0x100 [100787.483737] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.484768] blk_mq_flush_plug_list+0x135/0x220 [100787.485758] blk_flush_plug_list+0xd7/0x100 [100787.486696] blk_finish_plug+0x25/0x36 [100787.487569] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.488809] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [100787.489953] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.491260] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.492542] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.493666] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.495186] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.496120] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.497225] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.498292] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.499397] ? flush_work+0x42/0x1d0 [100787.500211] ? internal_add_timer+0x42/0x70 [100787.501111] ? _cond_resched+0x15/0x30 [100787.501956] ? mutex_lock+0xe/0x30 [100787.502742] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.503905] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.505174] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.506508] ? lu_context_exit+0x41/0x70 [obdclass] [100787.507636] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.508699] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.509868] kthread+0x134/0x150 [100787.510625] ? set_kthread_struct+0x50/0x50 [100787.511536] ret_from_fork+0x1f/0x40 [100787.512373] CPU: 7 PID: 527166 Comm: ll_ost_io01_060 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.515029] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.516675] Call Trace: [100787.517289] dump_stack+0x41/0x60 [100787.517985] warn_alloc.cold.127+0x7b/0x108 [100787.518800] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.519665] ? __alloc_pages_nodemask+0x166/0x330 [100787.520528] __alloc_pages_nodemask+0x2e2/0x330 [100787.521369] kmalloc_order+0x28/0x90 [100787.522061] kmalloc_order_trace+0x1d/0xb0 [100787.522837] __kmalloc+0x203/0x250 [100787.523599] ? __wake_up_common+0x7a/0x190 [100787.524394] virtqueue_add+0x493/0xc70 [100787.525123] ? finish_wait+0x80/0x80 [100787.525816] virtqueue_add_sgs+0x80/0xa0 [100787.526559] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.527508] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.528313] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.529229] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.530251] scsi_queue_rq+0x512/0xb10 [100787.530974] __blk_mq_try_issue_directly+0x163/0x200 [100787.531880] blk_mq_request_issue_directly+0x4e/0xb0 [100787.532767] blk_mq_try_issue_list_directly+0x62/0x100 [100787.533687] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.534569] blk_mq_flush_plug_list+0x135/0x220 [100787.535408] blk_flush_plug_list+0xd7/0x100 [100787.536187] blk_finish_plug+0x25/0x36 [100787.536903] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.537944] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.538932] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.540075] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.541159] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.542107] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.543025] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.543824] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.544757] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.545658] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.546593] ? flush_work+0x42/0x1d0 [100787.547299] ? internal_add_timer+0x42/0x70 [100787.548080] ? _cond_resched+0x15/0x30 [100787.548800] ? mutex_lock+0xe/0x30 [100787.549465] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.550468] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.551575] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.552688] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.553591] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.554566] kthread+0x134/0x150 [100787.555212] ? set_kthread_struct+0x50/0x50 [100787.555989] ret_from_fork+0x1f/0x40 [100787.556702] CPU: 2 PID: 527273 Comm: ll_ost_io00_034 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.558906] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.560467] Call Trace: [100787.561016] dump_stack+0x41/0x60 [100787.561699] warn_alloc.cold.127+0x7b/0x108 [100787.562562] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.563462] ? __alloc_pages_nodemask+0x166/0x330 [100787.564355] __alloc_pages_nodemask+0x2e2/0x330 [100787.565198] kmalloc_order+0x28/0x90 [100787.565905] kmalloc_order_trace+0x1d/0xb0 [100787.566694] __kmalloc+0x203/0x250 [100787.567389] ? __wake_up_common+0x7a/0x190 [100787.568164] virtqueue_add+0x493/0xc70 [100787.568904] ? finish_wait+0x80/0x80 [100787.569818] virtqueue_add_sgs+0x80/0xa0 [100787.570871] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.572283] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.573474] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.574455] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.575462] scsi_queue_rq+0x512/0xb10 [100787.576199] __blk_mq_try_issue_directly+0x163/0x200 [100787.577102] blk_mq_request_issue_directly+0x4e/0xb0 [100787.578017] blk_mq_try_issue_list_directly+0x62/0x100 [100787.578950] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.579841] blk_mq_flush_plug_list+0x135/0x220 [100787.580697] blk_flush_plug_list+0xd7/0x100 [100787.581499] blk_finish_plug+0x25/0x36 [100787.582247] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.583308] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.584291] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.585567] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.586857] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.587832] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.588742] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.589555] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.590544] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.591472] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.592418] ? apic_timer_interrupt+0xa/0x20 [100787.593225] ? _cond_resched+0x15/0x30 [100787.593959] ? mutex_lock+0xe/0x30 [100787.594638] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.595616] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.596678] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.597786] ? finish_wait+0x80/0x80 [100787.598528] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.599458] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.600451] kthread+0x134/0x150 [100787.601096] ? set_kthread_struct+0x50/0x50 [100787.601885] ret_from_fork+0x1f/0x40 [100787.602593] CPU: 1 PID: 527434 Comm: ll_ost_io00_057 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.605601] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.607586] Call Trace: [100787.608514] dump_stack+0x41/0x60 [100787.609327] warn_alloc.cold.127+0x7b/0x108 [100787.610322] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.611709] ? __alloc_pages_nodemask+0x166/0x330 [100787.612836] __alloc_pages_nodemask+0x2e2/0x330 [100787.613875] kmalloc_order+0x28/0x90 [100787.614756] kmalloc_order_trace+0x1d/0xb0 [100787.615736] __kmalloc+0x203/0x250 [100787.616597] virtqueue_add+0x493/0xc70 [100787.617526] ? finish_wait+0x80/0x80 [100787.618442] virtqueue_add_sgs+0x80/0xa0 [100787.619403] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.620670] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.621699] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.622901] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.624215] scsi_queue_rq+0x512/0xb10 [100787.625059] __blk_mq_try_issue_directly+0x163/0x200 [100787.626126] blk_mq_request_issue_directly+0x4e/0xb0 [100787.627209] blk_mq_try_issue_list_directly+0x62/0x100 [100787.628289] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.629348] blk_mq_flush_plug_list+0x135/0x220 [100787.630343] blk_flush_plug_list+0xd7/0x100 [100787.631270] blk_finish_plug+0x25/0x36 [100787.632135] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.633377] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.634508] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.635812] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.637111] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.638277] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.639341] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.640288] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.641385] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.642447] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.643546] ? flush_work+0x42/0x1d0 [100787.644362] ? internal_add_timer+0x42/0x70 [100787.645337] ? _cond_resched+0x15/0x30 [100787.646177] ? mutex_lock+0xe/0x30 [100787.646955] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.648137] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.649424] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.650699] ? finish_wait+0x80/0x80 [100787.651521] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.652568] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.653707] kthread+0x134/0x150 [100787.654458] ? set_kthread_struct+0x50/0x50 [100787.655364] ret_from_fork+0x1f/0x40 [100787.656178] CPU: 8 PID: 527250 Comm: ll_ost_io02_045 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.659141] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.661285] Call Trace: [100787.662075] dump_stack+0x41/0x60 [100787.663021] warn_alloc.cold.127+0x7b/0x108 [100787.664159] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.665360] ? __alloc_pages_nodemask+0x166/0x330 [100787.666599] __alloc_pages_nodemask+0x2e2/0x330 [100787.667777] kmalloc_order+0x28/0x90 [100787.668760] kmalloc_order_trace+0x1d/0xb0 [100787.669829] __kmalloc+0x203/0x250 [100787.670785] virtqueue_add+0x493/0xc70 [100787.671805] ? finish_wait+0x80/0x80 [100787.672825] virtqueue_add_sgs+0x80/0xa0 [100787.673904] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.675255] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.676396] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.677694] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.679152] scsi_queue_rq+0x512/0xb10 [100787.680219] __blk_mq_try_issue_directly+0x163/0x200 [100787.681587] blk_mq_request_issue_directly+0x4e/0xb0 [100787.682935] blk_mq_try_issue_list_directly+0x62/0x100 [100787.684289] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.685731] blk_mq_flush_plug_list+0x135/0x220 [100787.686999] blk_flush_plug_list+0xd7/0x100 [100787.688148] blk_finish_plug+0x25/0x36 [100787.689233] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.690734] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.692092] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.693639] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.695166] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.696571] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.697855] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.698986] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.700301] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.701588] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.702955] ? flush_work+0x42/0x1d0 [100787.703966] ? internal_add_timer+0x42/0x70 [100787.705074] ? _cond_resched+0x15/0x30 [100787.706102] ? mutex_lock+0xe/0x30 [100787.707055] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.708538] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.710085] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.711730] ? finish_wait+0x80/0x80 [100787.712735] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.714044] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.715430] kthread+0x134/0x150 [100787.716351] ? set_kthread_struct+0x50/0x50 [100787.717452] ret_from_fork+0x1f/0x40 [100787.718459] CPU: 10 PID: 527206 Comm: ll_ost_io02_034 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100787.721230] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100787.723300] Call Trace: [100787.724120] dump_stack+0x41/0x60 [100787.725139] warn_alloc.cold.127+0x7b/0x108 [100787.726393] __alloc_pages_slowpath+0xcb2/0xcd0 [100787.727644] ? __alloc_pages_nodemask+0x166/0x330 [100787.728937] __alloc_pages_nodemask+0x2e2/0x330 [100787.730103] kmalloc_order+0x28/0x90 [100787.731103] kmalloc_order_trace+0x1d/0xb0 [100787.732235] __kmalloc+0x203/0x250 [100787.733160] virtqueue_add+0x493/0xc70 [100787.734204] ? finish_wait+0x80/0x80 [100787.735236] virtqueue_add_sgs+0x80/0xa0 [100787.736301] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100787.737681] ? scsi_alloc_sgtables+0x84/0x1c0 [100787.738861] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100787.740167] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100787.741579] scsi_queue_rq+0x512/0xb10 [100787.742625] __blk_mq_try_issue_directly+0x163/0x200 [100787.743895] blk_mq_request_issue_directly+0x4e/0xb0 [100787.745253] blk_mq_try_issue_list_directly+0x62/0x100 [100787.746567] blk_mq_sched_insert_requests+0xa4/0xf0 [100787.747839] blk_mq_flush_plug_list+0x135/0x220 [100787.749067] blk_flush_plug_list+0xd7/0x100 [100787.750216] blk_finish_plug+0x25/0x36 [100787.751256] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100787.752765] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100787.754155] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.755794] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100787.757383] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100787.758770] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100787.760052] ofd_commitrw+0x5b4/0xd20 [ofd] [100787.761185] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.762539] obd_commitrw+0x1b6/0x370 [ptlrpc] [100787.763841] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100787.765195] ? flush_work+0x42/0x1d0 [100787.766202] ? internal_add_timer+0x42/0x70 [100787.767319] ? _cond_resched+0x15/0x30 [100787.768395] ? mutex_lock+0xe/0x30 [100787.769382] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100787.770734] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100787.772293] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100787.773869] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100787.775166] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100787.776569] kthread+0x134/0x150 [100787.777479] ? set_kthread_struct+0x50/0x50 [100787.778704] ret_from_fork+0x1f/0x40 [100794.100542] warn_alloc: 37 callbacks suppressed [100794.100545] ll_ost_io04_069: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100794.100585] ll_ost_io02_011: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100794.100609] ll_ost_io02_047: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100794.100620] CPU: 10 PID: 527253 Comm: ll_ost_io02_047 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.100624] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.100626] Call Trace: [100794.100630] dump_stack+0x41/0x60 [100794.100638] warn_alloc.cold.127+0x7b/0x108 [100794.100644] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.100648] ? __alloc_pages_nodemask+0x166/0x330 [100794.100651] __alloc_pages_nodemask+0x2e2/0x330 [100794.100654] kmalloc_order+0x28/0x90 [100794.100660] kmalloc_order_trace+0x1d/0xb0 [100794.100664] __kmalloc+0x203/0x250 [100794.100669] virtqueue_add+0x493/0xc70 [100794.100675] ? finish_wait+0x80/0x80 [100794.100679] virtqueue_add_sgs+0x80/0xa0 [100794.100685] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.100690] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.100696] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.100700] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.100704] scsi_queue_rq+0x512/0xb10 [100794.100707] __blk_mq_try_issue_directly+0x163/0x200 [100794.100714] blk_mq_request_issue_directly+0x4e/0xb0 [100794.100717] blk_mq_try_issue_list_directly+0x62/0x100 [100794.100721] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.100725] blk_mq_flush_plug_list+0x135/0x220 [100794.100729] blk_flush_plug_list+0xd7/0x100 [100794.100735] blk_finish_plug+0x25/0x36 [100794.100740] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.100775] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.100809] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.100838] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.100867] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.100895] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.100917] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.100936] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.101075] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.101189] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.101306] ? flush_work+0x42/0x1d0 [100794.101313] ? internal_add_timer+0x42/0x70 [100794.101317] ? _cond_resched+0x15/0x30 [100794.101321] ? mutex_lock+0xe/0x30 [100794.101324] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.101432] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.101542] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.101672] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.101814] ,cpuset= [100794.101788] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.101893] kthread+0x134/0x150 [100794.101898] ? set_kthread_struct+0x50/0x50 [100794.101901] ret_from_fork+0x1f/0x40 [100794.101908] warn_alloc_show_mem: 9 callbacks suppressed [100794.101910] Mem-Info: [100794.101914] active_anon:27565 inactive_anon:151456 isolated_anon:0 active_file:26996122 inactive_file:6829947 isolated_file:3 unevictable:0 dirty:10359 writeback:0 slab_reclaimable:1019426 slab_unreclaimable:629498 mapped:22398 shmem:145 pagetables:5818 bounce:0 free:2357709 free_pcp:380 free_cma:0 [100794.101920] Node 0 active_anon:110260kB inactive_anon:605824kB active_file:107984488kB inactive_file:27319788kB unevictable:0kB isolated(anon):0kB isolated(file):12kB mapped:89592kB dirty:41436kB writeback:0kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35808kB pagetables:23272kB all_unreclaimable? no [100794.101928] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100794.101934] lowmem_reserve[]: 0 913 150007 150007 150007 [100794.101940] Node 0 DMA32 free:596456kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100794.101947] lowmem_reserve[]: 0 0 149093 149093 149093 [100794.101952] Node 0 Normal free:8823116kB min:49268kB low:201940kB high:354612kB active_anon:109880kB inactive_anon:605820kB active_file:107982080kB inactive_file:27321052kB unevictable:0kB writepending:41168kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1520kB local_pcp:36kB free_cma:0kB [100794.101958] lowmem_reserve[]: 0 0 0 0 0 [100794.101963] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [100794.101979] Node 0 DMA32: 16*4kB (UM) 19*8kB (UMH) 49*16kB (UME) 38*32kB (UMEH) 37*64kB (UM) 38*128kB (UM) 40*256kB (UM) 42*512kB (UMEH) 36*1024kB (UMH) 39*2048kB (UM) 107*4096kB (UM) = 596200kB [100794.102000] Node 0 Normal: 954301*4kB (UME) 546287*8kB (UME) 39667*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 8822172kB [100794.102017] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100794.102019] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100794.102021] 33825725 total pagecache pages [100794.102022] 1301 pages in swap cache [100794.102023] Swap cache stats: add 1751244, delete 1749969, find 101026/122176 [100794.102025] Free swap = 4905216kB [100794.102026] Total swap = 11075580kB [100794.102027] 39321462 pages RAM [100794.102028] 0 pages HighMem/MovableOnly [100794.102029] 905848 pages reserved [100794.102030] 0 pages hwpoisoned [100794.104465] ,cpuset= [100794.104500] ll_ost_io02_054: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100794.104508] CPU: 8 PID: 527318 Comm: ll_ost_io02_054 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.104511] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.104513] Call Trace: [100794.104516] dump_stack+0x41/0x60 [100794.104523] warn_alloc.cold.127+0x7b/0x108 [100794.104529] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.104532] ? __alloc_pages_nodemask+0x166/0x330 [100794.104535] __alloc_pages_nodemask+0x2e2/0x330 [100794.104538] kmalloc_order+0x28/0x90 [100794.104543] kmalloc_order_trace+0x1d/0xb0 [100794.104546] __kmalloc+0x203/0x250 [100794.104551] virtqueue_add+0x493/0xc70 [100794.104556] ? finish_wait+0x80/0x80 [100794.104559] virtqueue_add_sgs+0x80/0xa0 [100794.104563] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.104568] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.104572] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.104575] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.104578] scsi_queue_rq+0x512/0xb10 [100794.104580] __blk_mq_try_issue_directly+0x163/0x200 [100794.104586] blk_mq_request_issue_directly+0x4e/0xb0 [100794.104589] blk_mq_try_issue_list_directly+0x62/0x100 [100794.104593] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.104596] blk_mq_flush_plug_list+0x135/0x220 [100794.104600] blk_flush_plug_list+0xd7/0x100 [100794.104605] blk_finish_plug+0x25/0x36 [100794.104609] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.104647] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.104676] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.104701] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.104726] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.104750] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.104770] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.104787] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.104902] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.105000] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.105101] ? __slab_free+0x240/0x330 [100794.105105] ? flush_work+0x42/0x1d0 [100794.105109] ? internal_add_timer+0x42/0x70 [100794.105112] ? _cond_resched+0x15/0x30 [100794.105114] ? mutex_lock+0xe/0x30 [100794.105117] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.105220] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.105328] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.105437] ? lu_context_exit+0x41/0x70 [obdclass] [100794.105517] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.105626] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.105727] kthread+0x134/0x150 [100794.105731] ? set_kthread_struct+0x50/0x50 [100794.105734] ret_from_fork+0x1f/0x40 [100794.106905] / [100794.124482] ll_ost_io03_029: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100794.124716] ll_ost_io01_053: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [100794.124902] / [100794.124901] ,mems_allowed=0 [100794.124906] CPU: 17 PID: 528057 Comm: ll_ost_io04_069 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.124910] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.124911] Call Trace: [100794.124915] dump_stack+0x41/0x60 [100794.124923] warn_alloc.cold.127+0x7b/0x108 [100794.124929] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.124933] ? __alloc_pages_nodemask+0x166/0x330 [100794.124937] __alloc_pages_nodemask+0x2e2/0x330 [100794.124940] kmalloc_order+0x28/0x90 [100794.124945] kmalloc_order_trace+0x1d/0xb0 [100794.124948] __kmalloc+0x203/0x250 [100794.124953] virtqueue_add+0x493/0xc70 [100794.124960] ? finish_wait+0x80/0x80 [100794.124963] virtqueue_add_sgs+0x80/0xa0 [100794.124968] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.124973] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.124978] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.124982] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.124986] scsi_queue_rq+0x512/0xb10 [100794.124989] __blk_mq_try_issue_directly+0x163/0x200 [100794.124995] blk_mq_request_issue_directly+0x4e/0xb0 [100794.125000] blk_mq_try_issue_list_directly+0x62/0x100 [100794.125004] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.125008] blk_mq_flush_plug_list+0x135/0x220 [100794.125012] blk_flush_plug_list+0xd7/0x100 [100794.125018] blk_finish_plug+0x25/0x36 [100794.125023] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.125062] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.125094] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.125122] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.125151] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.125181] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.125200] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.125210] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.125340] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.125451] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.125574] ? flush_work+0x42/0x1d0 [100794.125580] ? internal_add_timer+0x42/0x70 [100794.125583] ? _cond_resched+0x15/0x30 [100794.125586] ? mutex_lock+0xe/0x30 [100794.125590] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.125700] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.125822] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.125935] ,cpuset= [100794.125932] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.126054] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.126170] kthread+0x134/0x150 [100794.126174] ? set_kthread_struct+0x50/0x50 [100794.126178] ret_from_fork+0x1f/0x40 [100794.127371] ,mems_allowed=0 [100794.127371] / [100794.127376] CPU: 9 PID: 527071 Comm: ll_ost_io02_011 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.127379] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.127381] Call Trace: [100794.127384] dump_stack+0x41/0x60 [100794.127391] warn_alloc.cold.127+0x7b/0x108 [100794.127397] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.127400] ? __alloc_pages_nodemask+0x166/0x330 [100794.127404] __alloc_pages_nodemask+0x2e2/0x330 [100794.127407] kmalloc_order+0x28/0x90 [100794.127412] kmalloc_order_trace+0x1d/0xb0 [100794.127415] __kmalloc+0x203/0x250 [100794.127420] virtqueue_add+0x493/0xc70 [100794.127427] ? finish_wait+0x80/0x80 [100794.127430] virtqueue_add_sgs+0x80/0xa0 [100794.127436] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.127441] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.127446] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.127450] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.127453] scsi_queue_rq+0x512/0xb10 [100794.127456] __blk_mq_try_issue_directly+0x163/0x200 [100794.127462] blk_mq_request_issue_directly+0x4e/0xb0 [100794.127466] blk_mq_try_issue_list_directly+0x62/0x100 [100794.127470] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.127473] blk_mq_flush_plug_list+0x135/0x220 [100794.127477] blk_flush_plug_list+0xd7/0x100 [100794.127483] blk_finish_plug+0x25/0x36 [100794.127487] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.127521] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.127554] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.127580] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.127608] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.127636] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.127660] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.127678] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.127802] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.127918] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.128029] ? __slab_free+0x240/0x330 [100794.128034] ? flush_work+0x42/0x1d0 [100794.128040] ? internal_add_timer+0x42/0x70 [100794.128044] ? _cond_resched+0x15/0x30 [100794.128047] ? mutex_lock+0xe/0x30 [100794.128052] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.128163] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.128280] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.128382] ll_ost_io00_079: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [100794.128391] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.128498] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.129816] ,mems_allowed=0 [100794.129817] / [100794.130507] ll_ost_io03_002: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [100794.131232] kthread+0x134/0x150 [100794.132263] ,mems_allowed=0 [100794.132264] / [100794.133550] ? set_kthread_struct+0x50/0x50 [100794.134653] / [100794.134652] ,mems_allowed=0 [100794.136027] ret_from_fork+0x1f/0x40 [100794.137117] ,mems_allowed=0 [100794.138411] CPU: 14 PID: 527229 Comm: ll_ost_io03_029 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.140297] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.141378] ll_ost_io00_018: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100794.384916] Call Trace: [100794.385559] dump_stack+0x41/0x60 [100794.386349] warn_alloc.cold.127+0x7b/0x108 [100794.387269] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.388244] ? __alloc_pages_nodemask+0x166/0x330 [100794.389246] __alloc_pages_nodemask+0x2e2/0x330 [100794.390223] kmalloc_order+0x28/0x90 [100794.391047] kmalloc_order_trace+0x1d/0xb0 [100794.391958] __kmalloc+0x203/0x250 [100794.392750] virtqueue_add+0x493/0xc70 [100794.393606] ? finish_wait+0x80/0x80 [100794.394428] virtqueue_add_sgs+0x80/0xa0 [100794.395307] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.396416] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.397364] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.398411] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.399570] scsi_queue_rq+0x512/0xb10 [100794.400428] __blk_mq_try_issue_directly+0x163/0x200 [100794.401482] blk_mq_request_issue_directly+0x4e/0xb0 [100794.402527] blk_mq_try_issue_list_directly+0x62/0x100 [100794.403589] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.404621] blk_mq_flush_plug_list+0x135/0x220 [100794.405596] blk_flush_plug_list+0xd7/0x100 [100794.406525] blk_finish_plug+0x25/0x36 [100794.407374] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.408588] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.409704] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.411000] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.412269] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.413382] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.414406] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.415325] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.416424] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.417492] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.418599] ? flush_work+0x42/0x1d0 [100794.419422] ? internal_add_timer+0x42/0x70 [100794.420329] ? _cond_resched+0x15/0x30 [100794.421168] ? mutex_lock+0xe/0x30 [100794.421952] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.423132] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.424421] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.425726] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.426792] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.427938] kthread+0x134/0x150 [100794.428697] ? set_kthread_struct+0x50/0x50 [100794.429603] ret_from_fork+0x1f/0x40 [100794.430413] CPU: 2 PID: 527784 Comm: ll_ost_io00_079 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.433230] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.434986] Call Trace: [100794.435598] dump_stack+0x41/0x60 [100794.436342] warn_alloc.cold.127+0x7b/0x108 [100794.437468] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.438686] ? __alloc_pages_nodemask+0x166/0x330 [100794.439933] __alloc_pages_nodemask+0x2e2/0x330 [100794.441121] kmalloc_order+0x28/0x90 [100794.441982] kmalloc_order_trace+0x1d/0xb0 [100794.442790] __kmalloc+0x203/0x250 [100794.443467] virtqueue_add+0x493/0xc70 [100794.444199] ? finish_wait+0x80/0x80 [100794.444944] virtqueue_add_sgs+0x80/0xa0 [100794.445718] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.446756] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.447615] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.448569] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.449584] scsi_queue_rq+0x512/0xb10 [100794.450316] __blk_mq_try_issue_directly+0x163/0x200 [100794.451211] blk_mq_request_issue_directly+0x4e/0xb0 [100794.452106] blk_mq_try_issue_list_directly+0x62/0x100 [100794.453019] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.453906] blk_mq_flush_plug_list+0x135/0x220 [100794.454793] blk_flush_plug_list+0xd7/0x100 [100794.455587] blk_finish_plug+0x25/0x36 [100794.456331] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.457419] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.458428] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.459611] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.460762] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.461729] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.462626] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.463519] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.464487] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.465428] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.466423] ? flush_work+0x42/0x1d0 [100794.467147] ? internal_add_timer+0x42/0x70 [100794.468044] ? _cond_resched+0x15/0x30 [100794.468793] ? mutex_lock+0xe/0x30 [100794.469509] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.470554] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.471685] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.472869] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.473792] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.474788] kthread+0x134/0x150 [100794.475458] ? set_kthread_struct+0x50/0x50 [100794.476279] ret_from_fork+0x1f/0x40 [100794.477016] CPU: 15 PID: 188108 Comm: ll_ost_io03_002 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.483901] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.487580] Call Trace: [100794.488237] dump_stack+0x41/0x60 [100794.488981] warn_alloc.cold.127+0x7b/0x108 [100794.489942] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.491104] ? __alloc_pages_nodemask+0x166/0x330 [100794.492317] __alloc_pages_nodemask+0x2e2/0x330 [100794.493524] kmalloc_order+0x28/0x90 [100794.494500] kmalloc_order_trace+0x1d/0xb0 [100794.495456] __kmalloc+0x203/0x250 [100794.496214] ? __wake_up_common+0x7a/0x190 [100794.497015] virtqueue_add+0x493/0xc70 [100794.497911] ? finish_wait+0x80/0x80 [100794.498634] virtqueue_add_sgs+0x80/0xa0 [100794.499417] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.500598] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.501696] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.502818] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.503887] scsi_queue_rq+0x512/0xb10 [100794.504813] __blk_mq_try_issue_directly+0x163/0x200 [100794.506055] blk_mq_request_issue_directly+0x4e/0xb0 [100794.507292] blk_mq_try_issue_list_directly+0x62/0x100 [100794.508565] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.509788] blk_mq_flush_plug_list+0x135/0x220 [100794.510938] blk_flush_plug_list+0xd7/0x100 [100794.511854] blk_finish_plug+0x25/0x36 [100794.512877] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.514017] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.515308] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.516746] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.517859] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.518995] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.520138] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.521133] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.522357] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.523601] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.524901] ? flush_work+0x42/0x1d0 [100794.525786] ? internal_add_timer+0x42/0x70 [100794.526726] ? _cond_resched+0x15/0x30 [100794.527728] ? mutex_lock+0xe/0x30 [100794.528662] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.530061] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.531564] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.533015] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.534092] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.535477] kthread+0x134/0x150 [100794.536388] ? set_kthread_struct+0x50/0x50 [100794.537230] ret_from_fork+0x1f/0x40 [100794.537973] CPU: 7 PID: 527157 Comm: ll_ost_io01_053 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.540193] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.541715] Call Trace: [100794.542261] dump_stack+0x41/0x60 [100794.542941] warn_alloc.cold.127+0x7b/0x108 [100794.543750] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.544591] ? __alloc_pages_nodemask+0x166/0x330 [100794.545457] __alloc_pages_nodemask+0x2e2/0x330 [100794.546312] kmalloc_order+0x28/0x90 [100794.547022] kmalloc_order_trace+0x1d/0xb0 [100794.547806] __kmalloc+0x203/0x250 [100794.548490] virtqueue_add+0x493/0xc70 [100794.549220] ? finish_wait+0x80/0x80 [100794.549922] virtqueue_add_sgs+0x80/0xa0 [100794.550673] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.551660] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.552483] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.553413] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.554424] scsi_queue_rq+0x512/0xb10 [100794.555139] __blk_mq_try_issue_directly+0x163/0x200 [100794.556030] blk_mq_request_issue_directly+0x4e/0xb0 [100794.557069] blk_mq_try_issue_list_directly+0x62/0x100 [100794.558189] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.559085] blk_mq_flush_plug_list+0x135/0x220 [100794.559919] blk_flush_plug_list+0xd7/0x100 [100794.560706] blk_finish_plug+0x25/0x36 [100794.561440] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.562482] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.563452] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.564621] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.565704] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.566659] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.567551] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.568361] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.569322] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.570243] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.571214] ? flush_work+0x42/0x1d0 [100794.571929] ? internal_add_timer+0x42/0x70 [100794.572736] ? _cond_resched+0x15/0x30 [100794.573478] ? mutex_lock+0xe/0x30 [100794.574161] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.575160] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.576262] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.577391] ? ofd_object_print+0x20/0x20 [ofd] [100794.578275] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.579222] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.580236] kthread+0x134/0x150 [100794.581008] ? set_kthread_struct+0x50/0x50 [100794.582060] ret_from_fork+0x1f/0x40 [100794.582800] CPU: 0 PID: 527189 Comm: ll_ost_io00_018 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100794.585479] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100794.587514] Call Trace: [100794.588296] dump_stack+0x41/0x60 [100794.589236] warn_alloc.cold.127+0x7b/0x108 [100794.590344] __alloc_pages_slowpath+0xcb2/0xcd0 [100794.591513] __alloc_pages_nodemask+0x2e2/0x330 [100794.592760] kmalloc_order+0x28/0x90 [100794.593737] kmalloc_order_trace+0x1d/0xb0 [100794.594813] __kmalloc+0x203/0x250 [100794.595750] virtqueue_add+0x493/0xc70 [100794.596739] ? finish_wait+0x80/0x80 [100794.597700] virtqueue_add_sgs+0x80/0xa0 [100794.598733] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100794.600039] ? scsi_alloc_sgtables+0x84/0x1c0 [100794.601148] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100794.602391] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100794.603774] scsi_queue_rq+0x512/0xb10 [100794.604776] __blk_mq_try_issue_directly+0x163/0x200 [100794.606002] blk_mq_request_issue_directly+0x4e/0xb0 [100794.607238] blk_mq_try_issue_list_directly+0x62/0x100 [100794.608518] blk_mq_sched_insert_requests+0xa4/0xf0 [100794.609744] blk_mq_flush_plug_list+0x135/0x220 [100794.610897] blk_flush_plug_list+0xd7/0x100 [100794.612018] blk_finish_plug+0x25/0x36 [100794.613027] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100794.614487] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100794.615830] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.617395] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100794.618949] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100794.620323] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100794.621607] ofd_commitrw+0x5b4/0xd20 [ofd] [100794.622744] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.624101] obd_commitrw+0x1b6/0x370 [ptlrpc] [100794.625416] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100794.626758] ? flush_work+0x42/0x1d0 [100794.627688] ? internal_add_timer+0x42/0x70 [100794.628488] ? _cond_resched+0x15/0x30 [100794.629298] ? mutex_lock+0xe/0x30 [100794.630224] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100794.631618] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100794.633152] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100794.634733] ? finish_wait+0x80/0x80 [100794.635564] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100794.636494] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100794.637486] kthread+0x134/0x150 [100794.638283] ? set_kthread_struct+0x50/0x50 [100794.639397] ret_from_fork+0x1f/0x40 [100802.238760] ll_ost_io04_037: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100802.241402] CPU: 19 PID: 527549 Comm: ll_ost_io04_037 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100802.243898] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100802.245663] Call Trace: [100802.246330] dump_stack+0x41/0x60 [100802.246934] ll_ost_io01_007: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100802.247138] warn_alloc.cold.127+0x7b/0x108 [100802.247315] ll_ost_io04_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100802.253497] __alloc_pages_slowpath+0xcb2/0xcd0 [100802.254549] __alloc_pages_nodemask+0x2e2/0x330 [100802.255604] kmalloc_order+0x28/0x90 [100802.256485] kmalloc_order_trace+0x1d/0xb0 [100802.257465] __kmalloc+0x203/0x250 [100802.258312] ? common_interrupt+0xa/0xf [100802.259242] virtqueue_add+0x493/0xc70 [100802.260173] ? finish_wait+0x80/0x80 [100802.261071] virtqueue_add_sgs+0x80/0xa0 [100802.262020] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100802.263204] ? scsi_alloc_sgtables+0x84/0x1c0 [100802.264203] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100802.265316] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100802.266565] scsi_queue_rq+0x512/0xb10 [100802.267448] __blk_mq_try_issue_directly+0x163/0x200 [100802.268540] blk_mq_request_issue_directly+0x4e/0xb0 [100802.269641] blk_mq_try_issue_list_directly+0x62/0x100 [100802.270759] blk_mq_sched_insert_requests+0xa4/0xf0 [100802.271840] blk_mq_flush_plug_list+0x135/0x220 [100802.272858] blk_flush_plug_list+0xd7/0x100 [100802.273802] blk_finish_plug+0x25/0x36 [100802.274673] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100802.275925] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100802.277088] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100802.278426] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100802.279795] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100802.281020] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100802.282138] ofd_commitrw+0x5b4/0xd20 [ofd] [100802.283172] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100802.284410] obd_commitrw+0x1b6/0x370 [ptlrpc] [100802.285606] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100802.286820] ? flush_work+0x42/0x1d0 [100802.287765] ? internal_add_timer+0x42/0x70 [100802.288773] ? _cond_resched+0x15/0x30 [100802.289732] ? mutex_lock+0xe/0x30 [100802.290596] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100802.291835] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100802.293196] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100802.294571] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100802.295692] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100802.296905] kthread+0x134/0x150 [100802.297691] ? set_kthread_struct+0x50/0x50 [100802.298624] ret_from_fork+0x1f/0x40 [100802.299481] warn_alloc_show_mem: 8 callbacks suppressed [100802.299483] Mem-Info: [100802.299482] CPU: 16 PID: 527998 Comm: ll_ost_io04_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100802.300601] active_anon:27537 inactive_anon:151624 isolated_anon:0 active_file:21262488 inactive_file:8305373 isolated_file:0 unevictable:0 dirty:31031 writeback:1 slab_reclaimable:1016509 slab_unreclaimable:673373 mapped:22060 shmem:145 pagetables:5818 bounce:0 free:6574718 free_pcp:396 free_cma:0 [100802.301452] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100802.303937] Node 0 active_anon:110148kB inactive_anon:606496kB active_file:85049952kB inactive_file:33221492kB unevictable:0kB isolated(anon):0kB isolated(file):0kB mapped:88240kB dirty:124124kB writeback:4kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35904kB pagetables:23272kB all_unreclaimable? no [100802.310226] Call Trace: [100802.312034] Node 0 [100802.317018] dump_stack+0x41/0x60 [100802.317739] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100802.318313] warn_alloc.cold.127+0x7b/0x108 [100802.319126] lowmem_reserve[]: 0 [100802.323771] __alloc_pages_slowpath+0xcb2/0xcd0 [100802.324718] 913 150007 [100802.325765] __alloc_pages_nodemask+0x2e2/0x330 [100802.326781] 150007 [100802.327744] kmalloc_order+0x28/0x90 [100802.328787] 150007 [100802.329650] kmalloc_order_trace+0x1d/0xb0 [100802.331105] __kmalloc+0x203/0x250 [100802.332126] Node 0 [100802.332621] virtqueue_add+0x493/0xc70 [100802.333548] DMA32 free:596536kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100802.334147] virtqueue_add_sgs+0x80/0xa0 [100802.335097] lowmem_reserve[]: [100802.339195] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100802.340201] 0 [100802.341159] ? scsi_alloc_sgtables+0x84/0x1c0 [100802.342406] 0 [100802.343142] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100802.344221] 149093 [100802.344774] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100802.345945] 149093 [100802.346532] scsi_queue_rq+0x512/0xb10 [100802.347802] 149093 [100802.348386] __blk_mq_try_issue_directly+0x163/0x200 [100802.349898] blk_mq_request_issue_directly+0x4e/0xb0 [100802.351031] Node 0 [100802.351630] blk_mq_try_issue_list_directly+0x62/0x100 [100802.352764] Normal free:25627220kB min:49268kB low:201940kB high:354612kB active_anon:109768kB inactive_anon:606492kB active_file:85115844kB inactive_file:33221488kB unevictable:0kB writepending:124128kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1168kB local_pcp:152kB free_cma:0kB [100802.353330] blk_mq_sched_insert_requests+0xa4/0xf0 [100802.354485] lowmem_reserve[]: 0 [100802.359679] blk_mq_flush_plug_list+0x135/0x220 [100802.360795] 0 [100802.361888] blk_flush_plug_list+0xd7/0x100 [100802.362919] 0 [100802.363711] blk_finish_plug+0x25/0x36 [100802.364699] 0 [100802.365505] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100802.366409] 0 [100802.367206] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100802.368955] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100802.370103] Node 0 [100802.370592] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100802.371942] DMA: [100802.372514] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100802.373823] 0*4kB [100802.374340] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100802.375514] 0*8kB [100802.376058] ofd_commitrw+0x5b4/0xd20 [ofd] [100802.377133] 0*16kB [100802.377679] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100802.378627] 0*32kB [100802.379181] obd_commitrw+0x1b6/0x370 [ptlrpc] [100802.380210] 0*64kB [100802.380811] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100802.381795] 0*128kB [100802.382351] ? flush_work+0x42/0x1d0 [100802.383404] 0*256kB [100802.384090] ? internal_add_timer+0x42/0x70 [100802.384943] 0*512kB [100802.385757] ? _cond_resched+0x15/0x30 [100802.386688] 1*1024kB [100802.387529] ? mutex_lock+0xe/0x30 [100802.388408] (U) [100802.389248] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100802.390063] 1*2048kB [100802.390833] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100802.391920] (M) [100802.392774] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100802.393974] 2*4096kB [100802.394756] ? osd_key_exit+0x5/0x100 [osd_ldiskfs] [100802.395954] (M) [100802.396567] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100802.397616] = 11264kB [100802.398153] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100802.399160] Node 0 [100802.399827] kthread+0x134/0x150 [100802.400950] DMA32: [100802.401739] ? set_kthread_struct+0x50/0x50 [100802.402565] 16*4kB [100802.403353] ret_from_fork+0x1f/0x40 [100802.404317] (UM) [100802.405108] CPU: 6 PID: 527017 Comm: ll_ost_io01_007 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100802.405935] 19*8kB [100802.406539] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100802.408945] (UMH) [100802.409769] Call Trace: [100802.411558] 56*16kB [100802.412360] dump_stack+0x41/0x60 [100802.413040] (UME) [100802.413835] warn_alloc.cold.127+0x7b/0x108 [100802.414688] 48*32kB [100802.415489] __alloc_pages_slowpath+0xcb2/0xcd0 [100802.416452] (UMEH) [100802.417278] __alloc_pages_nodemask+0x2e2/0x330 [100802.418315] 41*64kB [100802.419128] kmalloc_order+0x28/0x90 [100802.420147] (UMH) [100802.420724] kmalloc_order_trace+0x1d/0xb0 [100802.421607] 38*128kB [100802.422137] __kmalloc+0x203/0x250 [100802.423106] (UM) [100802.423885] virtqueue_add+0x493/0xc70 [100802.424709] 40*256kB [100802.425324] ? finish_wait+0x80/0x80 [100802.426207] (UM) [100802.426905] virtqueue_add_sgs+0x80/0xa0 [100802.427744] 42*512kB [100802.428416] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100802.429307] (UMEH) [100802.430144] ? scsi_alloc_sgtables+0x84/0x1c0 [100802.431277] 36*1024kB [100802.431853] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100802.432876] (UMH) [100802.433446] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100802.434576] 39*2048kB [100802.435374] scsi_queue_rq+0x512/0xb10 [100802.436593] (UM) [100802.437462] __blk_mq_try_issue_directly+0x163/0x200 [100802.438384] 107*4096kB [100802.439141] blk_mq_request_issue_directly+0x4e/0xb0 [100802.440259] (UM) [100802.440884] blk_mq_try_issue_list_directly+0x62/0x100 [100802.441992] = 596888kB [100802.442525] blk_mq_sched_insert_requests+0xa4/0xf0 [100802.443682] Node 0 [100802.444290] blk_mq_flush_plug_list+0x135/0x220 [100802.445401] Normal: [100802.446077] blk_flush_plug_list+0xd7/0x100 [100802.447130] 3574362*4kB [100802.447948] blk_finish_plug+0x25/0x36 [100802.448912] (UE) [100802.449780] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100802.450669] 1212567*8kB [100802.451443] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100802.452684] (UME) [100802.453314] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100802.454436] 95781*16kB [100802.454970] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100802.456279] (UMEH) [100802.456871] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100802.458147] 65*32kB [100802.458676] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100802.459798] (UEH) [100802.460357] ofd_commitrw+0x5b4/0xd20 [ofd] [100802.461383] 0*64kB [100802.462114] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100802.463030] 0*128kB [100802.463745] obd_commitrw+0x1b6/0x370 [ptlrpc] [100802.464738] 0*256kB [100802.465502] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100802.466451] 0*512kB [100802.467008] ? flush_work+0x42/0x1d0 [100802.468053] 0*1024kB [100802.468604] ? internal_add_timer+0x42/0x70 [100802.469457] 0*2048kB [100802.470259] ? _cond_resched+0x15/0x30 [100802.471193] 0*4096kB [100802.472003] ? mutex_lock+0xe/0x30 [100802.472889] = 25532560kB [100802.473473] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100802.474284] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100802.474946] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100802.476018] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100802.477527] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100802.478747] 29611796 total pagecache pages [100802.480215] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100802.481401] 1295 pages in swap cache [100802.482226] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100802.483211] Swap cache stats: add 1751516, delete 1750247, find 101124/122289 [100802.483967] kthread+0x134/0x150 [100802.485065] Free swap = 4905216kB [100802.486318] ? set_kthread_struct+0x50/0x50 [100802.487117] Total swap = 11075580kB [100802.487861] ret_from_fork+0x1f/0x40 [100802.488844] 39321462 pages RAM [100802.491267] 0 pages HighMem/MovableOnly [100802.492194] 905848 pages reserved [100802.493029] 0 pages hwpoisoned [100830.461055] ll_ost_io03_020: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100830.464099] CPU: 13 PID: 527059 Comm: ll_ost_io03_020 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100830.467170] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100830.469384] Call Trace: [100830.470294] dump_stack+0x41/0x60 [100830.471367] warn_alloc.cold.127+0x7b/0x108 [100830.472610] __alloc_pages_slowpath+0xcb2/0xcd0 [100830.473920] __alloc_pages_nodemask+0x2e2/0x330 [100830.475259] kmalloc_order+0x28/0x90 [100830.476368] kmalloc_order_trace+0x1d/0xb0 [100830.477590] __kmalloc+0x203/0x250 [100830.478656] ll_ost_io03_022: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100830.478691] virtqueue_add+0x493/0xc70 [100830.479027] ll_ost_io00_012: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100830.479784] ll_ost_io00_009: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100830.483054] ll_ost_io02_051: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100830.485865] virtqueue_add_sgs+0x80/0xa0 [100830.488799] ,cpuset= [100830.488945] ll_ost_io03_048: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100830.489826] ll_ost_io02_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100830.491308] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100830.492558] / [100830.493398] ? scsi_alloc_sgtables+0x84/0x1c0 [100830.495531] ,mems_allowed=0 [100830.498357] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100830.500359] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100830.504589] scsi_queue_rq+0x512/0xb10 [100830.505376] __blk_mq_try_issue_directly+0x163/0x200 [100830.506303] blk_mq_request_issue_directly+0x4e/0xb0 [100830.507246] blk_mq_try_issue_list_directly+0x62/0x100 [100830.508205] blk_mq_sched_insert_requests+0xa4/0xf0 [100830.509125] blk_mq_flush_plug_list+0x135/0x220 [100830.509988] blk_flush_plug_list+0xd7/0x100 [100830.510789] blk_finish_plug+0x25/0x36 [100830.511532] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100830.512587] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100830.513567] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.514722] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.515835] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100830.516821] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100830.517732] ofd_commitrw+0x5b4/0xd20 [ofd] [100830.518561] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.519525] obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.520490] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100830.521480] ? flush_work+0x42/0x1d0 [100830.522213] ? internal_add_timer+0x42/0x70 [100830.523051] ? _cond_resched+0x15/0x30 [100830.523813] ? mutex_lock+0xe/0x30 [100830.524550] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100830.525625] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100830.526760] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100830.527914] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100830.528846] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100830.529831] kthread+0x134/0x150 [100830.530492] ? set_kthread_struct+0x50/0x50 [100830.531288] ret_from_fork+0x1f/0x40 [100830.531997] warn_alloc_show_mem: 2 callbacks suppressed [100830.531999] Mem-Info: [100830.531998] CPU: 15 PID: 527325 Comm: ll_ost_io03_048 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100830.532937] active_anon:27537 inactive_anon:151804 isolated_anon:0 active_file:23370924 inactive_file:8158883 isolated_file:38 unevictable:0 dirty:8368 writeback:2 slab_reclaimable:1017556 slab_unreclaimable:686349 mapped:22445 shmem:145 pagetables:5820 bounce:0 free:4601107 free_pcp:484 free_cma:0 [100830.533642] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100830.535870] Node 0 active_anon:110148kB inactive_anon:607216kB active_file:93483696kB inactive_file:32635532kB unevictable:0kB isolated(anon):0kB isolated(file):152kB mapped:89780kB dirty:33472kB writeback:8kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35968kB pagetables:23280kB all_unreclaimable? no [100830.542756] Call Trace: [100830.544340] Node 0 [100830.550193] dump_stack+0x41/0x60 [100830.550802] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100830.551445] warn_alloc.cold.127+0x7b/0x108 [100830.552191] lowmem_reserve[]: [100830.556446] __alloc_pages_slowpath+0xcb2/0xcd0 [100830.557328] 0 [100830.558139] __alloc_pages_nodemask+0x2e2/0x330 [100830.559086] 913 [100830.559725] kmalloc_order+0x28/0x90 [100830.560686] 150007 [100830.561336] kmalloc_order_trace+0x1d/0xb0 [100830.562183] 150007 [100830.562876] __kmalloc+0x203/0x250 [100830.563777] 150007 [100830.564487] virtqueue_add+0x493/0xc70 [100830.566003] ? finish_wait+0x80/0x80 [100830.566810] Node 0 [100830.567383] virtqueue_add_sgs+0x80/0xa0 [100830.568162] DMA32 free:596392kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100830.568837] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100830.569675] lowmem_reserve[]: [100830.574080] ? scsi_alloc_sgtables+0x84/0x1c0 [100830.575146] 0 [100830.575973] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100830.576885] 0 [100830.577525] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100830.578546] 149093 [100830.579164] scsi_queue_rq+0x512/0xb10 [100830.580260] 149093 [100830.580937] __blk_mq_try_issue_directly+0x163/0x200 [100830.581740] 149093 [100830.582395] blk_mq_request_issue_directly+0x4e/0xb0 [100830.584061] blk_mq_try_issue_list_directly+0x62/0x100 [100830.585053] Node 0 [100830.585612] blk_mq_sched_insert_requests+0xa4/0xf0 [100830.586637] Normal free:17770468kB min:49268kB low:201940kB high:354612kB active_anon:109768kB inactive_anon:607212kB active_file:93511416kB inactive_file:32635432kB unevictable:0kB writepending:33480kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1960kB local_pcp:164kB free_cma:0kB [100830.587299] blk_mq_flush_plug_list+0x135/0x220 [100830.588276] lowmem_reserve[]: [100830.593333] blk_flush_plug_list+0xd7/0x100 [100830.594277] 0 [100830.595102] blk_finish_plug+0x25/0x36 [100830.595963] 0 [100830.596564] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100830.597368] 0 [100830.597951] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100830.599027] 0 [100830.599601] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.600599] 0 [100830.601169] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.602869] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100830.603992] Node 0 [100830.604540] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100830.605536] DMA: [100830.606194] ofd_commitrw+0x5b4/0xd20 [ofd] [100830.607148] 0*4kB [100830.607782] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.608656] 0*8kB [100830.609316] obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.610213] 0*16kB [100830.610853] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100830.611733] 0*32kB [100830.612401] ? flush_work+0x42/0x1d0 [100830.613342] 0*64kB [100830.613999] ? internal_add_timer+0x42/0x70 [100830.614847] 0*128kB [100830.615498] ? _cond_resched+0x15/0x30 [100830.616351] 0*256kB [100830.617024] ? mutex_lock+0xe/0x30 [100830.617817] 0*512kB [100830.618468] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100830.619181] 1*1024kB [100830.619838] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100830.620780] (U) [100830.621447] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100830.622475] 1*2048kB [100830.623078] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100830.624120] (M) [100830.624788] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100830.625649] 2*4096kB [100830.626255] kthread+0x134/0x150 [100830.627232] (M) [100830.627921] ? set_kthread_struct+0x50/0x50 [100830.628639] = 11264kB [100830.629240] ret_from_fork+0x1f/0x40 [100830.630098] Node 0 [100830.630753] CPU: 14 PID: 527061 Comm: ll_ost_io03_022 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100830.631738] DMA32: [100830.632460] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100830.634938] 16*4kB [100830.635717] Call Trace: [100830.637267] (UM) [100830.638052] dump_stack+0x41/0x60 [100830.638679] 19*8kB [100830.639421] warn_alloc.cold.127+0x7b/0x108 [100830.640350] (UM) [100830.641117] __alloc_pages_slowpath+0xcb2/0xcd0 [100830.642010] 57*16kB [100830.642722] __alloc_pages_nodemask+0x2e2/0x330 [100830.643653] (UME) [100830.644366] kmalloc_order+0x28/0x90 [100830.645229] 48*32kB [100830.645796] kmalloc_order_trace+0x1d/0xb0 [100830.646550] (UMEH) [100830.647118] __kmalloc+0x203/0x250 [100830.647932] 45*64kB [100830.648470] virtqueue_add+0x493/0xc70 [100830.649192] (UM) [100830.649744] ? finish_wait+0x80/0x80 [100830.650523] 38*128kB [100830.651055] virtqueue_add_sgs+0x80/0xa0 [100830.651802] (UMH) [100830.652376] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100830.653171] 41*256kB [100830.653690] ? scsi_alloc_sgtables+0x84/0x1c0 [100830.654689] (UMH) [100830.655250] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100830.656108] 42*512kB [100830.656629] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100830.657583] (UMEH) [100830.658157] scsi_queue_rq+0x512/0xb10 [100830.659179] 35*1024kB [100830.659722] __blk_mq_try_issue_directly+0x163/0x200 [100830.660476] (UMH) [100830.661072] blk_mq_request_issue_directly+0x4e/0xb0 [100830.662006] 39*2048kB [100830.662534] blk_mq_try_issue_list_directly+0x62/0x100 [100830.663481] (UM) [100830.664084] blk_mq_sched_insert_requests+0xa4/0xf0 [100830.665087] 107*4096kB [100830.665614] blk_mq_flush_plug_list+0x135/0x220 [100830.666542] (UM) [100830.667144] blk_flush_plug_list+0xd7/0x100 [100830.668030] = 596392kB [100830.668548] blk_finish_plug+0x25/0x36 [100830.669384] Node 0 [100830.669962] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100830.670707] Normal: [100830.671242] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100830.672331] 306116*4kB [100830.672879] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.673841] (UME) [100830.674426] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.675522] 1659160*8kB [100830.676044] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100830.677136] (UME) [100830.677743] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100830.678706] 203986*16kB [100830.679236] ofd_commitrw+0x5b4/0xd20 [ofd] [100830.680144] (UMEH) [100830.680736] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.681515] 21*32kB [100830.682065] obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.682950] (UMH) [100830.683512] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100830.684350] 0*64kB [100830.684888] ? flush_work+0x42/0x1d0 [100830.685791] 0*128kB [100830.686342] ? internal_add_timer+0x42/0x70 [100830.687087] 0*256kB [100830.687638] ? _cond_resched+0x15/0x30 [100830.688459] 0*512kB [100830.689032] ? mutex_lock+0xe/0x30 [100830.689800] 0*1024kB [100830.690354] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100830.691051] 0*2048kB [100830.691618] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100830.692532] 0*4096kB [100830.693119] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100830.694169] = 17762192kB [100830.694745] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100830.695794] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100830.696413] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100830.697249] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100830.698743] kthread+0x134/0x150 [100830.699708] 31542576 total pagecache pages [100830.701148] ? set_kthread_struct+0x50/0x50 [100830.701854] 1314 pages in swap cache [100830.702668] ret_from_fork+0x1f/0x40 [100830.703494] Swap cache stats: add 1751700, delete 1750412, find 101228/122412 [100830.704253] CPU: 11 PID: 527312 Comm: ll_ost_io02_051 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100830.705015] Free swap = 4905216kB [100830.706927] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100830.709284] Total swap = 11075580kB [100830.710269] Call Trace: [100830.713016] 39321462 pages RAM [100830.713965] dump_stack+0x41/0x60 [100830.714858] 0 pages HighMem/MovableOnly [100830.715589] warn_alloc.cold.127+0x7b/0x108 [100830.717078] 905848 pages reserved [100830.718009] __alloc_pages_slowpath+0xcb2/0xcd0 [100830.719319] 0 pages hwpoisoned [100830.720119] ? __alloc_pages_nodemask+0x166/0x330 [100830.723508] __alloc_pages_nodemask+0x2e2/0x330 [100830.724682] kmalloc_order+0x28/0x90 [100830.725721] kmalloc_order_trace+0x1d/0xb0 [100830.726836] __kmalloc+0x203/0x250 [100830.727805] virtqueue_add+0x493/0xc70 [100830.728821] ? finish_wait+0x80/0x80 [100830.729818] virtqueue_add_sgs+0x80/0xa0 [100830.730892] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100830.732285] ? scsi_alloc_sgtables+0x84/0x1c0 [100830.733442] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100830.734735] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100830.736165] scsi_queue_rq+0x512/0xb10 [100830.737228] __blk_mq_try_issue_directly+0x163/0x200 [100830.738555] blk_mq_request_issue_directly+0x4e/0xb0 [100830.739861] blk_mq_try_issue_list_directly+0x62/0x100 [100830.741156] blk_mq_sched_insert_requests+0xa4/0xf0 [100830.742406] blk_mq_flush_plug_list+0x135/0x220 [100830.743618] blk_flush_plug_list+0xd7/0x100 [100830.744745] blk_finish_plug+0x25/0x36 [100830.745760] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100830.747257] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100830.748597] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.750168] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.751697] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100830.753038] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100830.754329] ofd_commitrw+0x5b4/0xd20 [ofd] [100830.755497] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.756828] obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.758169] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100830.759570] ? flush_work+0x42/0x1d0 [100830.760587] ? internal_add_timer+0x42/0x70 [100830.761720] ? _cond_resched+0x15/0x30 [100830.762756] ? mutex_lock+0xe/0x30 [100830.763728] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100830.765133] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100830.766702] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100830.768310] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100830.769617] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100830.771018] kthread+0x134/0x150 [100830.771918] ? set_kthread_struct+0x50/0x50 [100830.773016] ret_from_fork+0x1f/0x40 [100830.774011] CPU: 8 PID: 527133 Comm: ll_ost_io02_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100830.777007] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100830.779197] Call Trace: [100830.780063] dump_stack+0x41/0x60 [100830.781078] warn_alloc.cold.127+0x7b/0x108 [100830.782266] __alloc_pages_slowpath+0xcb2/0xcd0 [100830.783507] __alloc_pages_nodemask+0x2e2/0x330 [100830.784749] kmalloc_order+0x28/0x90 [100830.785827] kmalloc_order_trace+0x1d/0xb0 [100830.786985] __kmalloc+0x203/0x250 [100830.788023] virtqueue_add+0x493/0xc70 [100830.789146] virtqueue_add_sgs+0x80/0xa0 [100830.790328] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100830.791751] ? scsi_alloc_sgtables+0x84/0x1c0 [100830.793018] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100830.794419] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100830.795916] scsi_queue_rq+0x512/0xb10 [100830.797056] __blk_mq_try_issue_directly+0x163/0x200 [100830.798410] blk_mq_request_issue_directly+0x4e/0xb0 [100830.799757] blk_mq_try_issue_list_directly+0x62/0x100 [100830.801120] blk_mq_sched_insert_requests+0xa4/0xf0 [100830.802411] blk_mq_flush_plug_list+0x135/0x220 [100830.803574] blk_flush_plug_list+0xd7/0x100 [100830.804774] blk_finish_plug+0x25/0x36 [100830.805891] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100830.807399] ? clean_bdev_aliases+0x1a/0x1d0 [100830.808590] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.810205] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.811779] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100830.813179] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100830.814488] ofd_commitrw+0x5b4/0xd20 [ofd] [100830.815651] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.817017] obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.818357] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100830.819739] ? flush_work+0x42/0x1d0 [100830.820783] ? internal_add_timer+0x42/0x70 [100830.821919] ? _cond_resched+0x15/0x30 [100830.822967] ? mutex_lock+0xe/0x30 [100830.823974] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100830.825511] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100830.827119] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100830.828706] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100830.830034] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100830.831490] kthread+0x134/0x150 [100830.832444] ? set_kthread_struct+0x50/0x50 [100830.833590] ret_from_fork+0x1f/0x40 [100830.834602] CPU: 3 PID: 527127 Comm: ll_ost_io00_009 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100830.836907] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100830.838446] Call Trace: [100830.839040] dump_stack+0x41/0x60 [100830.839780] warn_alloc.cold.127+0x7b/0x108 [100830.840686] __alloc_pages_slowpath+0xcb2/0xcd0 [100830.841640] ? __alloc_pages_nodemask+0x166/0x330 [100830.842587] __alloc_pages_nodemask+0x2e2/0x330 [100830.843451] kmalloc_order+0x28/0x90 [100830.844184] kmalloc_order_trace+0x1d/0xb0 [100830.845009] __kmalloc+0x203/0x250 [100830.845715] virtqueue_add+0x493/0xc70 [100830.846452] ? finish_wait+0x80/0x80 [100830.847183] virtqueue_add_sgs+0x80/0xa0 [100830.847997] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100830.849014] ? scsi_alloc_sgtables+0x84/0x1c0 [100830.849912] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100830.851040] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100830.852141] scsi_queue_rq+0x512/0xb10 [100830.852940] __blk_mq_try_issue_directly+0x163/0x200 [100830.854122] blk_mq_request_issue_directly+0x4e/0xb0 [100830.855079] blk_mq_try_issue_list_directly+0x62/0x100 [100830.856065] blk_mq_sched_insert_requests+0xa4/0xf0 [100830.856983] blk_mq_flush_plug_list+0x135/0x220 [100830.857888] blk_flush_plug_list+0xd7/0x100 [100830.858731] blk_finish_plug+0x25/0x36 [100830.859484] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100830.860564] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100830.861543] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.862715] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.863821] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100830.864791] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100830.865715] ofd_commitrw+0x5b4/0xd20 [ofd] [100830.866527] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.867471] obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.868387] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100830.869360] ? flush_work+0x42/0x1d0 [100830.870093] ? internal_add_timer+0x42/0x70 [100830.870868] ? _cond_resched+0x15/0x30 [100830.871598] ? mutex_lock+0xe/0x30 [100830.872313] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100830.873331] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100830.874429] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100830.875537] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100830.876467] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100830.877447] kthread+0x134/0x150 [100830.878105] ? set_kthread_struct+0x50/0x50 [100830.878875] ret_from_fork+0x1f/0x40 [100830.879591] CPU: 0 PID: 527130 Comm: ll_ost_io00_012 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100830.882336] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100830.884444] Call Trace: [100830.885046] dump_stack+0x41/0x60 [100830.885791] warn_alloc.cold.127+0x7b/0x108 [100830.886648] __alloc_pages_slowpath+0xcb2/0xcd0 [100830.887583] __alloc_pages_nodemask+0x2e2/0x330 [100830.888491] kmalloc_order+0x28/0x90 [100830.889243] kmalloc_order_trace+0x1d/0xb0 [100830.890046] __kmalloc+0x203/0x250 [100830.890841] virtqueue_add+0x493/0xc70 [100830.891608] ? finish_wait+0x80/0x80 [100830.892320] virtqueue_add_sgs+0x80/0xa0 [100830.893080] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100830.894037] ? scsi_alloc_sgtables+0x84/0x1c0 [100830.894932] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100830.895995] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100830.897043] scsi_queue_rq+0x512/0xb10 [100830.898118] __blk_mq_try_issue_directly+0x163/0x200 [100830.899422] blk_mq_request_issue_directly+0x4e/0xb0 [100830.900693] blk_mq_try_issue_list_directly+0x62/0x100 [100830.901703] blk_mq_sched_insert_requests+0xa4/0xf0 [100830.902820] blk_mq_flush_plug_list+0x135/0x220 [100830.903796] blk_flush_plug_list+0xd7/0x100 [100830.904598] blk_finish_plug+0x25/0x36 [100830.905324] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100830.906377] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100830.907332] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.908423] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100830.909490] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100830.910422] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100830.911304] ofd_commitrw+0x5b4/0xd20 [ofd] [100830.912138] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.913184] obd_commitrw+0x1b6/0x370 [ptlrpc] [100830.914183] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100830.915168] ? flush_work+0x42/0x1d0 [100830.915893] ? internal_add_timer+0x42/0x70 [100830.916698] ? _cond_resched+0x15/0x30 [100830.917436] ? mutex_lock+0xe/0x30 [100830.918126] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100830.919144] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100830.920375] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100830.921513] ? finish_wait+0x80/0x80 [100830.922249] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100830.923195] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100830.924210] kthread+0x134/0x150 [100830.924873] ? set_kthread_struct+0x50/0x50 [100830.925691] ret_from_fork+0x1f/0x40 [100832.069117] ll_ost_io00_041: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100832.069129] ll_ost_io04_013: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [100832.069144] ll_ost_io04_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [100832.069156] CPU: 17 PID: 527998 Comm: ll_ost_io04_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100832.069160] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100832.069162] Call Trace: [100832.069167] dump_stack+0x41/0x60 [100832.069176] warn_alloc.cold.127+0x7b/0x108 [100832.069184] __alloc_pages_slowpath+0xcb2/0xcd0 [100832.069189] __alloc_pages_nodemask+0x2e2/0x330 [100832.069193] kmalloc_order+0x28/0x90 [100832.069199] kmalloc_order_trace+0x1d/0xb0 [100832.069203] __kmalloc+0x203/0x250 [100832.069209] virtqueue_add+0x493/0xc70 [100832.069216] virtqueue_add_sgs+0x80/0xa0 [100832.069221] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100832.069228] ? scsi_alloc_sgtables+0x84/0x1c0 [100832.069234] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100832.069238] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100832.069242] scsi_queue_rq+0x512/0xb10 [100832.069246] __blk_mq_try_issue_directly+0x163/0x200 [100832.069253] blk_mq_request_issue_directly+0x4e/0xb0 [100832.069258] blk_mq_try_issue_list_directly+0x62/0x100 [100832.069262] blk_mq_sched_insert_requests+0xa4/0xf0 [100832.069267] blk_mq_flush_plug_list+0x135/0x220 [100832.069271] blk_flush_plug_list+0xd7/0x100 [100832.069277] blk_finish_plug+0x25/0x36 [100832.069284] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100832.069326] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100832.069361] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100832.069390] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100832.069419] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100832.069447] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100832.069462] ofd_commitrw+0x5b4/0xd20 [ofd] [100832.069474] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100832.069598] obd_commitrw+0x1b6/0x370 [ptlrpc] [100832.069712] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100832.069819] ? set_next_entity+0xb5/0x1e0 [100832.069825] ? internal_add_timer+0x42/0x70 [100832.069830] ? _cond_resched+0x15/0x30 [100832.069833] ? mutex_lock+0xe/0x30 [100832.069838] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100832.069962] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100832.070091] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100832.070207] ? finish_wait+0x80/0x80 [100832.070211] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100832.070323] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100832.070438] kthread+0x134/0x150 [100832.070442] ? set_kthread_struct+0x50/0x50 [100832.070445] ret_from_fork+0x1f/0x40 [100832.070451] warn_alloc_show_mem: 6 callbacks suppressed [100832.070453] Mem-Info: [100832.070456] active_anon:27537 inactive_anon:151783 isolated_anon:0 active_file:23817186 inactive_file:7866202 isolated_file:7 unevictable:0 dirty:8864 writeback:0 slab_reclaimable:1016856 slab_unreclaimable:691637 mapped:22669 shmem:145 pagetables:5820 bounce:0 free:4442608 free_pcp:403 free_cma:0 [100832.070463] Node 0 active_anon:110148kB inactive_anon:607132kB active_file:95268744kB inactive_file:31464808kB unevictable:0kB isolated(anon):0kB isolated(file):28kB mapped:90676kB dirty:35456kB writeback:0kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35968kB pagetables:23280kB all_unreclaimable? no [100832.070471] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100832.070478] lowmem_reserve[]: 0 913 150007 150007 150007 [100832.070484] Node 0 DMA32 free:596488kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [100832.070490] lowmem_reserve[]: 0 0 149093 149093 149093 [100832.070494] Node 0 Normal free:17162680kB min:49268kB low:201940kB high:354612kB active_anon:109768kB inactive_anon:607128kB active_file:95269352kB inactive_file:31465292kB unevictable:0kB writepending:35456kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1588kB local_pcp:8kB free_cma:0kB [100832.070502] lowmem_reserve[]: 0 0 0 0 0 [100832.070506] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [100832.070523] Node 0 DMA32: 16*4kB (UM) 19*8kB (UM) 57*16kB (UME) 45*32kB (UMEH) 45*64kB (UM) 38*128kB (UMH) 41*256kB (UMH) 42*512kB (UMEH) 35*1024kB (UMH) 39*2048kB (UM) 107*4096kB (UM) = 596296kB [100832.070546] Node 0 Normal: 131764*4kB (UME) 1665760*8kB (UME) 206916*16kB (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 17163792kB [100832.070563] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [100832.070565] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [100832.070567] 31683169 total pagecache pages [100832.070568] 1293 pages in swap cache [100832.070569] Swap cache stats: add 1751701, delete 1750434, find 101228/122412 [100832.070571] Free swap = 4905216kB [100832.070572] Total swap = 11075580kB [100832.070573] 39321462 pages RAM [100832.070574] 0 pages HighMem/MovableOnly [100832.070575] 905848 pages reserved [100832.070576] 0 pages hwpoisoned [100832.071977] CPU: 1 PID: 527355 Comm: ll_ost_io00_041 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100832.074640] ,cpuset= [100832.077532] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100832.080720] / [100832.082602] Call Trace: [100832.083553] ,mems_allowed=0 [100832.084412] dump_stack+0x41/0x60 [100832.086850] warn_alloc.cold.127+0x7b/0x108 [100832.192276] __alloc_pages_slowpath+0xcb2/0xcd0 [100832.193343] ? __alloc_pages_nodemask+0x166/0x330 [100832.194499] __alloc_pages_nodemask+0x2e2/0x330 [100832.195580] kmalloc_order+0x28/0x90 [100832.196506] kmalloc_order_trace+0x1d/0xb0 [100832.197507] __kmalloc+0x203/0x250 [100832.198423] virtqueue_add+0x493/0xc70 [100832.199369] ? finish_wait+0x80/0x80 [100832.200263] virtqueue_add_sgs+0x80/0xa0 [100832.201254] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100832.202462] ? scsi_alloc_sgtables+0x84/0x1c0 [100832.203521] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100832.204686] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100832.205934] scsi_queue_rq+0x512/0xb10 [100832.206847] __blk_mq_try_issue_directly+0x163/0x200 [100832.208112] blk_mq_request_issue_directly+0x4e/0xb0 [100832.209374] blk_mq_try_issue_list_directly+0x62/0x100 [100832.210595] blk_mq_sched_insert_requests+0xa4/0xf0 [100832.211732] blk_mq_flush_plug_list+0x135/0x220 [100832.212764] blk_flush_plug_list+0xd7/0x100 [100832.213724] blk_finish_plug+0x25/0x36 [100832.214610] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100832.215882] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [100832.217055] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100832.218401] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100832.219709] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100832.220864] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100832.221932] ofd_commitrw+0x5b4/0xd20 [ofd] [100832.222880] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100832.223975] obd_commitrw+0x1b6/0x370 [ptlrpc] [100832.225032] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100832.226179] ? flush_work+0x42/0x1d0 [100832.227013] ? internal_add_timer+0x42/0x70 [100832.227939] ? _cond_resched+0x15/0x30 [100832.228783] ? mutex_lock+0xe/0x30 [100832.229570] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100832.230743] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100832.232032] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100832.233316] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100832.234356] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100832.235492] kthread+0x134/0x150 [100832.236256] ? set_kthread_struct+0x50/0x50 [100832.237169] ret_from_fork+0x1f/0x40 [100832.237972] CPU: 19 PID: 527210 Comm: ll_ost_io04_013 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [100832.240906] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [100832.242978] Call Trace: [100832.243755] dump_stack+0x41/0x60 [100832.244678] warn_alloc.cold.127+0x7b/0x108 [100832.245791] __alloc_pages_slowpath+0xcb2/0xcd0 [100832.246969] ? __alloc_pages_nodemask+0x166/0x330 [100832.248208] __alloc_pages_nodemask+0x2e2/0x330 [100832.249384] kmalloc_order+0x28/0x90 [100832.250373] kmalloc_order_trace+0x1d/0xb0 [100832.251449] __kmalloc+0x203/0x250 [100832.252395] virtqueue_add+0x493/0xc70 [100832.253422] ? finish_wait+0x80/0x80 [100832.254414] virtqueue_add_sgs+0x80/0xa0 [100832.255471] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [100832.256792] ? scsi_alloc_sgtables+0x84/0x1c0 [100832.257920] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [100832.259195] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [100832.260571] scsi_queue_rq+0x512/0xb10 [100832.261593] __blk_mq_try_issue_directly+0x163/0x200 [100832.262861] blk_mq_request_issue_directly+0x4e/0xb0 [100832.264140] blk_mq_try_issue_list_directly+0x62/0x100 [100832.265464] blk_mq_sched_insert_requests+0xa4/0xf0 [100832.266732] blk_mq_flush_plug_list+0x135/0x220 [100832.267912] blk_flush_plug_list+0xd7/0x100 [100832.269030] blk_finish_plug+0x25/0x36 [100832.270094] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [100832.271553] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100832.273107] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [100832.274627] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [100832.275961] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [100832.277218] ofd_commitrw+0x5b4/0xd20 [ofd] [100832.278332] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [100832.279670] obd_commitrw+0x1b6/0x370 [ptlrpc] [100832.280946] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [100832.282271] ? flush_work+0x42/0x1d0 [100832.283268] ? internal_add_timer+0x42/0x70 [100832.284378] ? _cond_resched+0x15/0x30 [100832.285407] ? mutex_lock+0xe/0x30 [100832.286383] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [100832.287846] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [100832.289423] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [100832.290996] ptlrpc_main+0xbec/0x1530 [ptlrpc] [100832.292302] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [100832.293737] kthread+0x134/0x150 [100832.294685] ? set_kthread_struct+0x50/0x50 [100832.295819] ret_from_fork+0x1f/0x40 [102090.899258] warn_alloc: 382 callbacks suppressed [102090.899260] ll_ost_io01_063: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102090.902949] CPU: 4 PID: 527200 Comm: ll_ost_io01_063 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102090.905053] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102090.906650] Call Trace: [102090.907278] dump_stack+0x41/0x60 [102090.908111] warn_alloc.cold.127+0x7b/0x108 [102090.909073] __alloc_pages_slowpath+0xcb2/0xcd0 [102090.910262] ? __alloc_pages_nodemask+0x166/0x330 [102090.911506] __alloc_pages_nodemask+0x2e2/0x330 [102090.912719] kmalloc_order+0x28/0x90 [102090.913719] kmalloc_order_trace+0x1d/0xb0 [102090.914692] __kmalloc+0x203/0x250 [102090.915388] virtqueue_add+0x493/0xc70 [102090.916153] ? finish_wait+0x80/0x80 [102090.916932] virtqueue_add_sgs+0x80/0xa0 [102090.917777] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102090.918832] ? scsi_alloc_sgtables+0x84/0x1c0 [102090.919659] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102090.920645] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102090.921657] scsi_queue_rq+0x512/0xb10 [102090.922427] __blk_mq_try_issue_directly+0x163/0x200 [102090.923367] blk_mq_request_issue_directly+0x4e/0xb0 [102090.924298] blk_mq_try_issue_list_directly+0x62/0x100 [102090.925267] blk_mq_sched_insert_requests+0xa4/0xf0 [102090.926263] blk_mq_flush_plug_list+0x135/0x220 [102090.927127] blk_flush_plug_list+0xd7/0x100 [102090.927945] blk_finish_plug+0x25/0x36 [102090.928752] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102090.930059] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102090.931114] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102090.932241] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102090.933349] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102090.934363] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102090.935335] ofd_commitrw+0x5b4/0xd20 [ofd] [102090.936170] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102090.937290] obd_commitrw+0x1b6/0x370 [ptlrpc] [102090.938341] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102090.939408] ? flush_work+0x42/0x1d0 [102090.940194] ? internal_add_timer+0x42/0x70 [102090.941045] ? _cond_resched+0x15/0x30 [102090.941808] ? mutex_lock+0xe/0x30 [102090.942553] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102090.943570] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102090.944672] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102090.945782] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102090.946743] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102090.947960] kthread+0x134/0x150 [102090.948877] ? set_kthread_struct+0x50/0x50 [102090.949991] ret_from_fork+0x1f/0x40 [102090.950776] warn_alloc_show_mem: 2 callbacks suppressed [102090.950785] Mem-Info: [102090.952222] active_anon:27845 inactive_anon:129868 isolated_anon:0 active_file:25672393 inactive_file:9028345 isolated_file:3 unevictable:0 dirty:9321 writeback:2 slab_reclaimable:1019397 slab_unreclaimable:655415 mapped:18405 shmem:145 pagetables:5861 bounce:0 free:1479642 free_pcp:593 free_cma:0 [102090.959466] Node 0 active_anon:111380kB inactive_anon:519472kB active_file:102702156kB inactive_file:36084180kB unevictable:0kB isolated(anon):0kB isolated(file):12kB mapped:73620kB dirty:37284kB writeback:8kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:36000kB pagetables:23444kB all_unreclaimable? no [102090.965745] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102090.969698] lowmem_reserve[]: 0 913 150007 150007 150007 [102090.970705] Node 0 DMA32 free:596552kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102090.975103] lowmem_reserve[]: 0 0 149093 149093 149093 [102090.976461] Node 0 Normal free:5355988kB min:49268kB low:201940kB high:354612kB active_anon:111000kB inactive_anon:519468kB active_file:102717040kB inactive_file:36042636kB unevictable:0kB writepending:37292kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2272kB local_pcp:132kB free_cma:0kB [102090.982481] lowmem_reserve[]: 0 0 0 0 0 [102090.983549] Node 0 DMA: 0*4kB 0*8kB 0*16kB 0*32kB 0*64kB 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB (M) 2*4096kB (M) = 11264kB [102090.985451] Node 0 DMA32: 16*4kB (UMH) 19*8kB (UMH) 61*16kB (UMEH) 44*32kB (UMEH) 36*64kB (UM) 38*128kB (UM) 38*256kB (UM) 39*512kB (UMEH) 32*1024kB (UMH) 36*2048kB (UM) 110*4096kB (UM) = 596520kB [102090.988456] Node 0 Normal: 679095*4kB (UMEH) 300373*8kB (UMEH) 14977*16kB (UMEH) 4*32kB (H) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 5359124kB [102090.991046] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [102090.992895] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [102090.994687] 34687835 total pagecache pages [102090.995542] 1608 pages in swap cache [102090.996349] Swap cache stats: add 1809501, delete 1807911, find 108024/130903 [102090.997646] Free swap = 4731952kB [102090.998415] Total swap = 11075580kB [102090.999213] 39321462 pages RAM [102090.999997] 0 pages HighMem/MovableOnly [102091.001006] 905848 pages reserved [102091.002020] 0 pages hwpoisoned [102093.164201] ll_ost_io01_014: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102093.164201] ll_ost_io02_006: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [102093.164294] ll_ost_io01_071: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102093.164306] CPU: 6 PID: 527331 Comm: ll_ost_io01_071 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.164309] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.164310] Call Trace: [102093.164314] dump_stack+0x41/0x60 [102093.164322] warn_alloc.cold.127+0x7b/0x108 [102093.164328] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.164331] __alloc_pages_nodemask+0x2e2/0x330 [102093.164334] kmalloc_order+0x28/0x90 [102093.164340] kmalloc_order_trace+0x1d/0xb0 [102093.164342] __kmalloc+0x203/0x250 [102093.164348] virtqueue_add+0x493/0xc70 [102093.164353] ? finish_wait+0x80/0x80 [102093.164357] virtqueue_add_sgs+0x80/0xa0 [102093.164362] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.164367] ll_ost_io02_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102093.164368] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.164373] ,cpuset=/,mems_allowed=0 [102093.164375] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.164379] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.164383] scsi_queue_rq+0x512/0xb10 [102093.164386] __blk_mq_try_issue_directly+0x163/0x200 [102093.164393] blk_mq_request_issue_directly+0x4e/0xb0 [102093.164397] blk_mq_try_issue_list_directly+0x62/0x100 [102093.164401] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.164406] blk_mq_flush_plug_list+0x135/0x220 [102093.164410] blk_flush_plug_list+0xd7/0x100 [102093.164416] blk_finish_plug+0x25/0x36 [102093.164422] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.164461] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.164489] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.164518] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.164546] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.164572] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.164590] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.164733] ll_ost_io04_042: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102093.164714] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.164845] ll_ost_io04_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102093.164827] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.164934] ? flush_work+0x42/0x1d0 [102093.164940] ? internal_add_timer+0x42/0x70 [102093.164944] ? _cond_resched+0x15/0x30 [102093.164947] ? mutex_lock+0xe/0x30 [102093.164951] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.165073] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.165195] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.165313] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.165386] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.165490] kthread+0x134/0x150 [102093.165494] ? set_kthread_struct+0x50/0x50 [102093.165496] ret_from_fork+0x1f/0x40 [102093.165501] Mem-Info: [102093.165502] CPU: 17 PID: 527216 Comm: ll_ost_io04_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.165504] active_anon:27846 inactive_anon:129837 isolated_anon:0 active_file:25636504 inactive_file:9007877 isolated_file:2 unevictable:0 dirty:17201 writeback:0 slab_reclaimable:1019724 slab_unreclaimable:637984 mapped:19075 shmem:145 pagetables:5904 bounce:0 free:1553582 free_pcp:548 free_cma:0 [102093.165507] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.165509] Call Trace: [102093.165511] Node 0 active_anon:111384kB inactive_anon:519348kB active_file:102546016kB inactive_file:36031508kB unevictable:0kB isolated(anon):0kB isolated(file):8kB mapped:76300kB dirty:68804kB writeback:0kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35904kB pagetables:23616kB all_unreclaimable? no [102093.165513] dump_stack+0x41/0x60 [102093.165518] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102093.165520] warn_alloc.cold.127+0x7b/0x108 [102093.165526] lowmem_reserve[]: 0 913 [102093.165526] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.165528] 150007 150007 150007 [102093.165533] Node 0 [102093.165532] __alloc_pages_nodemask+0x2e2/0x330 [102093.165534] DMA32 free:596392kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102093.165536] kmalloc_order+0x28/0x90 [102093.165540] lowmem_reserve[]: 0 0 149093 149093 [102093.165542] kmalloc_order_trace+0x1d/0xb0 [102093.165543] 149093 [102093.165545] Node 0 Normal free:5607176kB min:49268kB low:201940kB high:354612kB active_anon:111004kB inactive_anon:519344kB active_file:102546100kB inactive_file:36031032kB unevictable:0kB writepending:68640kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:2196kB local_pcp:160kB free_cma:0kB [102093.165546] __kmalloc+0x203/0x250 [102093.165552] lowmem_reserve[]: 0 0 0 0 0 [102093.165552] virtqueue_add+0x493/0xc70 [102093.165558] Node 0 DMA: 0*4kB [102093.165559] ? finish_wait+0x80/0x80 [102093.165561] 0*8kB 0*16kB 0*32kB 0*64kB [102093.165563] virtqueue_add_sgs+0x80/0xa0 [102093.165566] 0*128kB 0*256kB 0*512kB 1*1024kB (U) 1*2048kB [102093.165569] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.165572] (M) 2*4096kB (M) = 11264kB [102093.165576] Node 0 DMA32: 16*4kB [102093.165575] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.165579] (UMH) 19*8kB (UMH) 61*16kB (UMEH) [102093.165581] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.165582] 38*32kB (UMEH) 37*64kB (UMH) 38*128kB (UM) [102093.165586] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.165587] 39*256kB (UMH) 38*512kB (UME) 32*1024kB [102093.165590] scsi_queue_rq+0x512/0xb10 [102093.165592] (UMH) 36*2048kB (UM) 110*4096kB (UM) = 596136kB [102093.165596] Node 0 Normal: [102093.165594] __blk_mq_try_issue_directly+0x163/0x200 [102093.165598] 670248*4kB (UMEH) 333182*8kB (UMEH) 16327*16kB [102093.165601] blk_mq_request_issue_directly+0x4e/0xb0 [102093.165603] (UMEH) 0*32kB 0*64kB 0*128kB 0*256kB [102093.165605] blk_mq_try_issue_list_directly+0x62/0x100 [102093.165608] 0*512kB 0*1024kB 0*2048kB 0*4096kB [102093.165610] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.165612] = 5607680kB [102093.165614] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [102093.165614] blk_mq_flush_plug_list+0x135/0x220 [102093.165616] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [102093.165619] 34644052 total pagecache pages [102093.165620] 1608 pages in swap cache [102093.165621] Swap cache stats: add 1809595, delete 1808008, find 108024/130904 [102093.165620] blk_flush_plug_list+0xd7/0x100 [102093.165623] Free swap = 4731952kB [102093.165625] Total swap = 11075580kB [102093.165626] 39321462 pages RAM [102093.165625] blk_finish_plug+0x25/0x36 [102093.165626] 0 pages HighMem/MovableOnly [102093.165628] 905848 pages reserved [102093.165629] 0 pages hwpoisoned [102093.165631] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.165668] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.165699] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.165729] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.165758] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.165787] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.165807] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.165826] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.165955] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.166068] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.166177] ? flush_work+0x42/0x1d0 [102093.166182] ? internal_add_timer+0x42/0x70 [102093.166186] ? _cond_resched+0x15/0x30 [102093.166188] ? mutex_lock+0xe/0x30 [102093.166192] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.166302] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.166411] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.166521] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.166617] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.166689] kthread+0x134/0x150 [102093.166692] ? set_kthread_struct+0x50/0x50 [102093.166695] ret_from_fork+0x1f/0x40 [102093.166696] ,cpuset=/,mems_allowed=0 [102093.167190] ll_ost_io01_061: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102093.169477] / [102093.172425] CPU: 5 PID: 527198 Comm: ll_ost_io01_061 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.175647] ,mems_allowed=0 [102093.177592] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.179532] ll_ost_io00_071: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102093.179676] Call Trace: [102093.192220] ll_ost_io01_074: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102093.192235] dump_stack+0x41/0x60 [102093.193616] ,cpuset= [102093.194549] warn_alloc.cold.127+0x7b/0x108 [102093.195710] / [102093.196355] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.197501] ,mems_allowed=0 [102093.198514] ? __alloc_pages_nodemask+0x166/0x330 [102093.200750] __alloc_pages_nodemask+0x2e2/0x330 [102093.356254] kmalloc_order+0x28/0x90 [102093.357076] kmalloc_order_trace+0x1d/0xb0 [102093.357981] __kmalloc+0x203/0x250 [102093.358776] virtqueue_add+0x493/0xc70 [102093.359616] ? finish_wait+0x80/0x80 [102093.360431] virtqueue_add_sgs+0x80/0xa0 [102093.361317] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.362427] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.363369] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.364421] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.365560] scsi_queue_rq+0x512/0xb10 [102093.366400] __blk_mq_try_issue_directly+0x163/0x200 [102093.367438] blk_mq_request_issue_directly+0x4e/0xb0 [102093.368472] blk_mq_try_issue_list_directly+0x62/0x100 [102093.369533] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.370578] blk_mq_flush_plug_list+0x135/0x220 [102093.371555] blk_flush_plug_list+0xd7/0x100 [102093.372463] blk_finish_plug+0x25/0x36 [102093.373318] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.374530] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.375647] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.376932] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.378185] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.379287] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.380352] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.381315] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.382472] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.383617] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.384734] ? flush_work+0x42/0x1d0 [102093.385559] ? internal_add_timer+0x42/0x70 [102093.386485] ? _cond_resched+0x15/0x30 [102093.387332] ? mutex_lock+0xe/0x30 [102093.388121] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.389269] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.390530] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.391792] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.392838] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.393985] kthread+0x134/0x150 [102093.394732] ? set_kthread_struct+0x50/0x50 [102093.395624] ret_from_fork+0x1f/0x40 [102093.396409] CPU: 19 PID: 527679 Comm: ll_ost_io04_042 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.398481] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.399987] Call Trace: [102093.400550] dump_stack+0x41/0x60 [102093.401218] warn_alloc.cold.127+0x7b/0x108 [102093.401990] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.402821] ? __alloc_pages_nodemask+0x166/0x330 [102093.403669] __alloc_pages_nodemask+0x2e2/0x330 [102093.404567] kmalloc_order+0x28/0x90 [102093.405499] kmalloc_order_trace+0x1d/0xb0 [102093.406522] __kmalloc+0x203/0x250 [102093.407293] virtqueue_add+0x493/0xc70 [102093.408184] ? finish_wait+0x80/0x80 [102093.409069] virtqueue_add_sgs+0x80/0xa0 [102093.410092] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.411386] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.412480] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.413714] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.415071] scsi_queue_rq+0x512/0xb10 [102093.416086] __blk_mq_try_issue_directly+0x163/0x200 [102093.417313] blk_mq_request_issue_directly+0x4e/0xb0 [102093.418524] blk_mq_try_issue_list_directly+0x62/0x100 [102093.419580] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.420476] blk_mq_flush_plug_list+0x135/0x220 [102093.421307] blk_flush_plug_list+0xd7/0x100 [102093.422092] blk_finish_plug+0x25/0x36 [102093.422819] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.424063] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.425345] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.426461] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.427575] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.428552] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.429431] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.430318] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.431261] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.432314] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.433579] ? flush_work+0x42/0x1d0 [102093.434426] ? internal_add_timer+0x42/0x70 [102093.435222] ? _cond_resched+0x15/0x30 [102093.435939] ? mutex_lock+0xe/0x30 [102093.436606] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.437586] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.438980] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.440443] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.441346] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.442326] kthread+0x134/0x150 [102093.442964] ? set_kthread_struct+0x50/0x50 [102093.443738] ret_from_fork+0x1f/0x40 [102093.444559] CPU: 4 PID: 527067 Comm: ll_ost_io01_014 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.446996] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.448840] Call Trace: [102093.449493] dump_stack+0x41/0x60 [102093.450194] warn_alloc.cold.127+0x7b/0x108 [102093.451092] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.451974] ? virtqueue_add+0x959/0xc70 [102093.452848] __alloc_pages_nodemask+0x2e2/0x330 [102093.453776] kmalloc_order+0x28/0x90 [102093.454527] kmalloc_order_trace+0x1d/0xb0 [102093.455307] __kmalloc+0x203/0x250 [102093.456030] virtqueue_add+0x493/0xc70 [102093.456812] ? finish_wait+0x80/0x80 [102093.457581] virtqueue_add_sgs+0x80/0xa0 [102093.458408] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.459481] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.460399] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.461484] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.462612] scsi_queue_rq+0x512/0xb10 [102093.463411] __blk_mq_try_issue_directly+0x163/0x200 [102093.464354] blk_mq_request_issue_directly+0x4e/0xb0 [102093.465360] blk_mq_try_issue_list_directly+0x62/0x100 [102093.466343] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.467468] blk_mq_flush_plug_list+0x135/0x220 [102093.468722] blk_flush_plug_list+0xd7/0x100 [102093.469907] blk_finish_plug+0x25/0x36 [102093.471026] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.472555] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.473992] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.475641] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.477292] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.478706] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.479727] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.480545] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.481517] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.482447] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.483400] ? flush_work+0x42/0x1d0 [102093.484099] ? internal_add_timer+0x42/0x70 [102093.484886] ? _cond_resched+0x15/0x30 [102093.485630] ? mutex_lock+0xe/0x30 [102093.486418] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.487464] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.488636] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.489811] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.490731] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.491732] kthread+0x134/0x150 [102093.492406] ? set_kthread_struct+0x50/0x50 [102093.493203] ret_from_fork+0x1f/0x40 [102093.493901] CPU: 6 PID: 528143 Comm: ll_ost_io01_074 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.496800] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.498880] Call Trace: [102093.499699] dump_stack+0x41/0x60 [102093.500668] warn_alloc.cold.127+0x7b/0x108 [102093.501804] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.502977] __alloc_pages_nodemask+0x2e2/0x330 [102093.504146] kmalloc_order+0x28/0x90 [102093.505155] kmalloc_order_trace+0x1d/0xb0 [102093.506295] __kmalloc+0x203/0x250 [102093.507319] virtqueue_add+0x493/0xc70 [102093.508390] virtqueue_add_sgs+0x80/0xa0 [102093.509501] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.510839] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.511980] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.513284] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.514669] scsi_queue_rq+0x512/0xb10 [102093.515733] __blk_mq_try_issue_directly+0x163/0x200 [102093.516996] blk_mq_request_issue_directly+0x4e/0xb0 [102093.518269] blk_mq_try_issue_list_directly+0x62/0x100 [102093.519583] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.520819] blk_mq_flush_plug_list+0x135/0x220 [102093.521996] blk_flush_plug_list+0xd7/0x100 [102093.523110] blk_finish_plug+0x25/0x36 [102093.524176] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.525640] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.527078] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.528665] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.530203] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.531530] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.532691] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.533719] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.535027] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.536279] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.537573] ? flush_work+0x42/0x1d0 [102093.538529] ? internal_add_timer+0x42/0x70 [102093.539581] ? _cond_resched+0x15/0x30 [102093.540387] ? mutex_lock+0xe/0x30 [102093.541078] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.542091] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.543245] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.544354] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.545288] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.546295] kthread+0x134/0x150 [102093.546964] ? set_kthread_struct+0x50/0x50 [102093.547802] ret_from_fork+0x1f/0x40 [102093.548615] CPU: 11 PID: 527133 Comm: ll_ost_io02_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.551646] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.553757] Call Trace: [102093.554566] dump_stack+0x41/0x60 [102093.555500] warn_alloc.cold.127+0x7b/0x108 [102093.556597] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.557695] ? __alloc_pages_nodemask+0x166/0x330 [102093.558802] __alloc_pages_nodemask+0x2e2/0x330 [102093.559834] kmalloc_order+0x28/0x90 [102093.560796] kmalloc_order_trace+0x1d/0xb0 [102093.561847] __kmalloc+0x203/0x250 [102093.562766] virtqueue_add+0x493/0xc70 [102093.563778] ? finish_wait+0x80/0x80 [102093.564790] virtqueue_add_sgs+0x80/0xa0 [102093.565843] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.567153] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.568257] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.569480] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.570832] scsi_queue_rq+0x512/0xb10 [102093.571844] __blk_mq_try_issue_directly+0x163/0x200 [102093.573075] blk_mq_request_issue_directly+0x4e/0xb0 [102093.574304] blk_mq_try_issue_list_directly+0x62/0x100 [102093.575554] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.576748] blk_mq_flush_plug_list+0x135/0x220 [102093.577938] blk_flush_plug_list+0xd7/0x100 [102093.579087] blk_finish_plug+0x25/0x36 [102093.580141] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.581584] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.582837] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.584152] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.585386] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.586539] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.587718] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.588915] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.590145] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.591402] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.592692] ? flush_work+0x42/0x1d0 [102093.593653] ? internal_add_timer+0x42/0x70 [102093.594718] ? _cond_resched+0x15/0x30 [102093.595763] ? mutex_lock+0xe/0x30 [102093.596754] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.598116] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.599577] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.601102] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.602344] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.603697] kthread+0x134/0x150 [102093.604593] ? set_kthread_struct+0x50/0x50 [102093.605663] ret_from_fork+0x1f/0x40 [102093.606616] CPU: 9 PID: 527018 Comm: ll_ost_io02_006 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.609542] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.611697] Call Trace: [102093.612578] dump_stack+0x41/0x60 [102093.613617] warn_alloc.cold.127+0x7b/0x108 [102093.614806] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.616024] __alloc_pages_nodemask+0x2e2/0x330 [102093.617253] kmalloc_order+0x28/0x90 [102093.618298] kmalloc_order_trace+0x1d/0xb0 [102093.619421] __kmalloc+0x203/0x250 [102093.620415] virtqueue_add+0x493/0xc70 [102093.621483] ? finish_wait+0x80/0x80 [102093.622503] virtqueue_add_sgs+0x80/0xa0 [102093.623576] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.624930] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.626107] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.627342] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.628763] scsi_queue_rq+0x512/0xb10 [102093.629857] __blk_mq_try_issue_directly+0x163/0x200 [102093.631165] blk_mq_request_issue_directly+0x4e/0xb0 [102093.632502] blk_mq_try_issue_list_directly+0x62/0x100 [102093.633836] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.635096] blk_mq_flush_plug_list+0x135/0x220 [102093.636314] blk_flush_plug_list+0xd7/0x100 [102093.637441] blk_finish_plug+0x25/0x36 [102093.638515] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.640040] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.641491] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.643153] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.644766] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.646152] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.647454] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.648652] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.650030] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.651383] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.652762] ? flush_work+0x42/0x1d0 [102093.653796] ? internal_add_timer+0x42/0x70 [102093.654961] ? _cond_resched+0x15/0x30 [102093.656029] ? mutex_lock+0xe/0x30 [102093.657016] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.658467] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.660052] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.661668] ? tgt_key_exit+0xc/0x20 [ptlrpc] [102093.663023] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.664394] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.665890] kthread+0x134/0x150 [102093.666921] ? set_kthread_struct+0x50/0x50 [102093.668137] ret_from_fork+0x1f/0x40 [102093.669215] CPU: 0 PID: 527768 Comm: ll_ost_io00_071 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102093.672084] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102093.674120] Call Trace: [102093.674936] dump_stack+0x41/0x60 [102093.675882] warn_alloc.cold.127+0x7b/0x108 [102093.676971] __alloc_pages_slowpath+0xcb2/0xcd0 [102093.678159] __alloc_pages_nodemask+0x2e2/0x330 [102093.679348] kmalloc_order+0x28/0x90 [102093.680360] kmalloc_order_trace+0x1d/0xb0 [102093.681479] __kmalloc+0x203/0x250 [102093.682458] virtqueue_add+0x493/0xc70 [102093.683490] ? finish_wait+0x80/0x80 [102093.684498] virtqueue_add_sgs+0x80/0xa0 [102093.685318] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102093.686264] ? scsi_alloc_sgtables+0x84/0x1c0 [102093.687064] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102093.688012] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102093.689009] scsi_queue_rq+0x512/0xb10 [102093.689750] __blk_mq_try_issue_directly+0x163/0x200 [102093.690638] blk_mq_request_issue_directly+0x4e/0xb0 [102093.691537] blk_mq_try_issue_list_directly+0x62/0x100 [102093.692441] blk_mq_sched_insert_requests+0xa4/0xf0 [102093.693320] blk_mq_flush_plug_list+0x135/0x220 [102093.694177] blk_flush_plug_list+0xd7/0x100 [102093.694954] blk_finish_plug+0x25/0x36 [102093.695674] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102093.696719] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102093.697680] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.698824] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102093.699902] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102093.700839] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102093.701738] ofd_commitrw+0x5b4/0xd20 [ofd] [102093.702538] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.703484] obd_commitrw+0x1b6/0x370 [ptlrpc] [102093.704414] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102093.705346] ? flush_work+0x42/0x1d0 [102093.706031] ? internal_add_timer+0x42/0x70 [102093.706816] ? _cond_resched+0x15/0x30 [102093.707513] ? mutex_lock+0xe/0x30 [102093.708195] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102093.709195] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102093.710281] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102093.711374] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102093.712378] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102093.713380] kthread+0x134/0x150 [102093.714017] ? set_kthread_struct+0x50/0x50 [102093.714829] ret_from_fork+0x1f/0x40 [102096.689802] warn_alloc: 945 callbacks suppressed [102096.689806] ll_ost_io01_072: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102096.693441] CPU: 5 PID: 527341 Comm: ll_ost_io01_072 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102096.695829] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102096.697605] Call Trace: [102096.698294] dump_stack+0x41/0x60 [102096.699096] warn_alloc.cold.127+0x7b/0x108 [102096.700004] __alloc_pages_slowpath+0xcb2/0xcd0 [102096.701024] ? __alloc_pages_nodemask+0x166/0x330 [102096.702218] __alloc_pages_nodemask+0x2e2/0x330 [102096.703440] kmalloc_order+0x28/0x90 [102096.704465] kmalloc_order_trace+0x1d/0xb0 [102096.705564] __kmalloc+0x203/0x250 [102096.706590] ? __wake_up_common+0x7a/0x190 [102096.707710] virtqueue_add+0x493/0xc70 [102096.708702] ? finish_wait+0x80/0x80 [102096.709633] virtqueue_add_sgs+0x80/0xa0 [102096.710722] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102096.712133] ? scsi_alloc_sgtables+0x84/0x1c0 [102096.713292] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102096.714563] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102096.715853] scsi_queue_rq+0x512/0xb10 [102096.716720] __blk_mq_try_issue_directly+0x163/0x200 [102096.717804] blk_mq_request_issue_directly+0x4e/0xb0 [102096.718849] blk_mq_try_issue_list_directly+0x62/0x100 [102096.720090] blk_mq_sched_insert_requests+0xa4/0xf0 [102096.721070] blk_mq_flush_plug_list+0x135/0x220 [102096.721987] blk_flush_plug_list+0xd7/0x100 [102096.722831] blk_finish_plug+0x25/0x36 [102096.723599] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102096.724687] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102096.725754] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.726911] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.728038] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102096.729023] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102096.729954] ofd_commitrw+0x5b4/0xd20 [ofd] [102096.730803] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.731811] obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.732819] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102096.733864] ? newidle_balance+0x2b6/0x3b0 [102096.734780] ? internal_add_timer+0x42/0x70 [102096.735875] ? _cond_resched+0x15/0x30 [102096.736933] ? mutex_lock+0xe/0x30 [102096.737943] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102096.739402] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102096.740845] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102096.742038] ? finish_wait+0x80/0x80 [102096.742919] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102096.744187] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102096.745564] kthread+0x134/0x150 [102096.746482] ? set_kthread_struct+0x50/0x50 [102096.747589] ret_from_fork+0x1f/0x40 [102096.748580] warn_alloc_show_mem: 8 callbacks suppressed [102096.748583] Mem-Info: [102096.750631] active_anon:27845 inactive_anon:129363 isolated_anon:0 active_file:24797789 inactive_file:8837920 isolated_file:1 unevictable:0 dirty:52340 writeback:0 slab_reclaimable:1021478 slab_unreclaimable:652890 mapped:19491 shmem:145 pagetables:5862 bounce:0 free:2544883 free_pcp:477 free_cma:0 [102096.758466] Node 0 active_anon:111380kB inactive_anon:517452kB active_file:99159940kB inactive_file:35376848kB unevictable:0kB isolated(anon):0kB isolated(file):4kB mapped:77964kB dirty:209360kB writeback:0kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:36032kB pagetables:23448kB all_unreclaimable? no [102096.765196] Node 0 DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102096.765299] ll_ost_io01_012: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102096.769698] lowmem_reserve[]: [102096.769699] ,cpuset=/ [102096.772013] 0 [102096.772981] ,mems_allowed=0 [102096.773738] 913 [102096.775298] 150007 [102096.776051] CPU: 7 PID: 527065 Comm: ll_ost_io01_012 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102096.776679] 150007 [102096.777429] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102096.779983] 150007 [102096.780805] Call Trace: [102096.783612] dump_stack+0x41/0x60 [102096.784383] Node 0 [102096.785070] warn_alloc.cold.127+0x7b/0x108 [102096.785960] DMA32 free:596360kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102096.786753] __alloc_pages_slowpath+0xcb2/0xcd0 [102096.787932] lowmem_reserve[]: [102096.792910] __alloc_pages_nodemask+0x2e2/0x330 [102096.794132] 0 [102096.795038] kmalloc_order+0x28/0x90 [102096.796147] 0 [102096.796878] kmalloc_order_trace+0x1d/0xb0 [102096.797837] 149093 [102096.798523] __kmalloc+0x203/0x250 [102096.799686] 149093 [102096.800535] virtqueue_add+0x493/0xc70 [102096.801450] 149093 [102096.802272] ? finish_wait+0x80/0x80 [102096.804002] virtqueue_add_sgs+0x80/0xa0 [102096.804914] Node 0 [102096.805514] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102096.806477] Normal free:9583308kB min:49268kB low:201940kB high:354612kB active_anon:111000kB inactive_anon:517448kB active_file:99164920kB inactive_file:35363740kB unevictable:0kB writepending:209560kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1612kB local_pcp:96kB free_cma:0kB [102096.807271] ? scsi_alloc_sgtables+0x84/0x1c0 [102096.808540] lowmem_reserve[]: [102096.814137] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102096.815283] 0 [102096.816239] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102096.817424] 0 [102096.818142] scsi_queue_rq+0x512/0xb10 [102096.819343] 0 [102096.820007] blk_mq_dispatch_rq_list+0x154/0x880 [102096.820934] 0 [102096.821632] blk_mq_do_dispatch_ctx+0x17d/0x1e0 [102096.822823] 0 [102096.823535] __blk_mq_sched_dispatch_requests+0x110/0x140 [102096.825103] blk_mq_sched_dispatch_requests+0x30/0x60 [102096.826454] Node 0 [102096.827093] __blk_mq_run_hw_queue+0x2b/0x70 [102096.828191] DMA: [102096.828962] __blk_mq_delay_run_hw_queue+0x141/0x160 [102096.830050] 0*4kB [102096.830844] blk_mq_sched_insert_requests+0x6c/0xf0 [102096.832094] 0*8kB [102096.832888] blk_mq_flush_plug_list+0x135/0x220 [102096.834152] 0*16kB [102096.834875] blk_flush_plug_list+0xd7/0x100 [102096.836002] 0*32kB [102096.836740] blk_finish_plug+0x25/0x36 [102096.837828] 0*64kB [102096.838538] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102096.839545] 0*128kB [102096.840289] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102096.841451] 0*256kB [102096.842164] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.843212] 0*512kB [102096.843869] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.845131] 1*1024kB [102096.845859] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102096.847187] (U) [102096.847899] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102096.848994] 1*2048kB [102096.849611] ofd_commitrw+0x5b4/0xd20 [ofd] [102096.850581] (M) [102096.851208] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.852073] 2*4096kB [102096.852751] obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.853710] (M) [102096.854361] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102096.855308] = 11264kB [102096.855903] ? flush_work+0x42/0x1d0 [102096.857072] Node 0 [102096.857821] ? internal_add_timer+0x42/0x70 [102096.858699] DMA32: [102096.859394] ? _cond_resched+0x15/0x30 [102096.860427] 16*4kB [102096.861172] ? mutex_lock+0xe/0x30 [102096.862129] (UMH) [102096.862876] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102096.863848] 19*8kB [102096.864564] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102096.865838] (UMH) [102096.866548] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102096.867887] 61*16kB [102096.868436] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102096.869434] (UMEH) [102096.870028] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102096.870837] 38*32kB [102096.871532] kthread+0x134/0x150 [102096.872474] (UMEH) [102096.873112] ? set_kthread_struct+0x50/0x50 [102096.873801] 36*64kB [102096.874389] ret_from_fork+0x1f/0x40 [102096.875208] (UM) [102096.876164] ll_ost_io01_063: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102096.876163] ll_ost_io02_062: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102096.876180] CPU: 9 PID: 527337 Comm: ll_ost_io02_062 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102096.876184] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102096.876186] Call Trace: [102096.876189] dump_stack+0x41/0x60 [102096.876196] warn_alloc.cold.127+0x7b/0x108 [102096.876203] __alloc_pages_slowpath+0xcb2/0xcd0 [102096.876207] ? __alloc_pages_nodemask+0x166/0x330 [102096.876210] __alloc_pages_nodemask+0x2e2/0x330 [102096.876214] kmalloc_order+0x28/0x90 [102096.876220] kmalloc_order_trace+0x1d/0xb0 [102096.876224] __kmalloc+0x203/0x250 [102096.876230] virtqueue_add+0x493/0xc70 [102096.876236] ? finish_wait+0x80/0x80 [102096.876241] virtqueue_add_sgs+0x80/0xa0 [102096.876246] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102096.876251] ? scsi_alloc_sgtables+0x84/0x1c0 [102096.876258] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102096.876262] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102096.876266] scsi_queue_rq+0x512/0xb10 [102096.876269] __blk_mq_try_issue_directly+0x163/0x200 [102096.876276] blk_mq_request_issue_directly+0x4e/0xb0 [102096.876280] blk_mq_try_issue_list_directly+0x62/0x100 [102096.876284] blk_mq_sched_insert_requests+0xa4/0xf0 [102096.876288] blk_mq_flush_plug_list+0x135/0x220 [102096.876292] blk_flush_plug_list+0xd7/0x100 [102096.876298] blk_finish_plug+0x25/0x36 [102096.876303] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102096.876338] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102096.876370] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.876397] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.876426] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102096.876453] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102096.876476] ofd_commitrw+0x5b4/0xd20 [ofd] [102096.876494] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.876664] 40*128kB [102096.876624] obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.876735] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102096.876847] ? flush_work+0x42/0x1d0 [102096.876852] ? internal_add_timer+0x42/0x70 [102096.876856] ? _cond_resched+0x15/0x30 [102096.876858] ? mutex_lock+0xe/0x30 [102096.876862] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102096.876971] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102096.877081] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102096.877188] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102096.877247] ,cpuset= [102096.877294] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102096.877406] kthread+0x134/0x150 [102096.877410] ? set_kthread_struct+0x50/0x50 [102096.877414] ret_from_fork+0x1f/0x40 [102096.879274] (UMH) [102096.881645] / [102096.884152] 39*256kB [102096.886054] ,mems_allowed=0 [102096.886761] (UMH) [102096.888605] 38*512kB [102096.889620] CPU: 4 PID: 527200 Comm: ll_ost_io01_063 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102096.890705] (UME) [102096.891710] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102096.892558] 32*1024kB (UMH) 36*2048kB [102096.893453] Call Trace: [102096.894238] (UM) [102096.895069] dump_stack+0x41/0x60 [102096.895883] 110*4096kB [102096.896735] warn_alloc.cold.127+0x7b/0x108 [102096.897862] (UM) [102096.898831] __alloc_pages_slowpath+0xcb2/0xcd0 [102096.898836] ? __alloc_pages_nodemask+0x166/0x330 [102096.898840] __alloc_pages_nodemask+0x2e2/0x330 [102096.898843] kmalloc_order+0x28/0x90 [102096.898849] kmalloc_order_trace+0x1d/0xb0 [102096.898852] __kmalloc+0x203/0x250 [102096.898857] virtqueue_add+0x493/0xc70 [102096.898863] virtqueue_add_sgs+0x80/0xa0 [102096.898867] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102096.898872] ? scsi_alloc_sgtables+0x84/0x1c0 [102096.898878] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102096.898881] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102096.898884] scsi_queue_rq+0x512/0xb10 [102096.898887] __blk_mq_try_issue_directly+0x163/0x200 [102096.898893] blk_mq_request_issue_directly+0x4e/0xb0 [102096.898896] blk_mq_try_issue_list_directly+0x62/0x100 [102096.898900] blk_mq_sched_insert_requests+0xa4/0xf0 [102096.898904] blk_mq_flush_plug_list+0x135/0x220 [102096.898907] blk_flush_plug_list+0xd7/0x100 [102096.898912] blk_finish_plug+0x25/0x36 [102096.898917] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102096.900021] = 596328kB [102096.901190] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102096.902045] Node 0 [102096.903074] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.904099] Normal: [102096.905155] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102096.906165] 1345162*4kB [102096.907124] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102096.907244] ll_ost_io03_017: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [102096.907248] ll_ost_io03_060: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102096.907251] /,mems_allowed=0 [102096.907252] ,cpuset= [102096.907254] /,mems_allowed=0 [102096.908046] (UEH) [102096.908893] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102096.910134] 476530*8kB [102096.911215] ofd_commitrw+0x5b4/0xd20 [ofd] [102096.912454] (UEH) [102096.913706] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.914813] 22302*16kB [102096.915680] ll_ost_io02_060: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102096.915705] ll_ost_io00_047: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset= [102096.915709] ll_ost_io01_040: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102096.915711] /,mems_allowed=0 [102096.915714] ,cpuset=/,mems_allowed=0 [102096.915858] obd_commitrw+0x1b6/0x370 [ptlrpc] [102096.916849] (UMEH) [102096.917877] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102096.918564] 0*32kB [102096.919593] ? flush_work+0x42/0x1d0 [102096.920654] 0*64kB [102096.921531] ? internal_add_timer+0x42/0x70 [102096.922497] 0*128kB [102096.923366] ? _cond_resched+0x15/0x30 [102096.924180] 0*256kB [102096.925254] ? mutex_lock+0xe/0x30 [102096.926464] 0*512kB [102096.927692] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102096.928699] 0*1024kB [102096.929324] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102096.930437] 0*2048kB [102096.931219] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102096.932181] 0*4096kB [102096.933037] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102096.933650] = 9549720kB [102096.934181] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102096.934823] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [102096.935525] kthread+0x134/0x150 [102096.936099] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [102096.936586] ? set_kthread_struct+0x50/0x50 [102096.937191] 33634624 total pagecache pages [102096.939536] ret_from_fork+0x1f/0x40 [102096.940099] 1608 pages in swap cache [102096.941838] CPU: 9 PID: 527335 Comm: ll_ost_io02_060 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102096.942809] Swap cache stats: add 1810069, delete 1808488, find 108024/130904 [102096.943475] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102096.944058] Free swap = 4729640kB [102096.944922] Call Trace: [102096.945532] Total swap = 11075580kB [102096.946403] dump_stack+0x41/0x60 [102096.946934] 39321462 pages RAM [102096.947850] warn_alloc.cold.127+0x7b/0x108 [102096.948766] 0 pages HighMem/MovableOnly [102096.949687] __alloc_pages_slowpath+0xcb2/0xcd0 [102096.950437] 905848 pages reserved [102096.953215] __alloc_pages_nodemask+0x2e2/0x330 [102096.957759] 0 pages hwpoisoned [102096.958749] kmalloc_order+0x28/0x90 [102097.038165] kmalloc_order_trace+0x1d/0xb0 [102097.039006] __kmalloc+0x203/0x250 [102097.039716] virtqueue_add+0x493/0xc70 [102097.040520] ? finish_wait+0x80/0x80 [102097.041288] virtqueue_add_sgs+0x80/0xa0 [102097.042075] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102097.043085] ? scsi_alloc_sgtables+0x84/0x1c0 [102097.043974] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102097.044946] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102097.046002] scsi_queue_rq+0x512/0xb10 [102097.046762] __blk_mq_try_issue_directly+0x163/0x200 [102097.047723] blk_mq_request_issue_directly+0x4e/0xb0 [102097.048672] blk_mq_try_issue_list_directly+0x62/0x100 [102097.049668] blk_mq_sched_insert_requests+0xa4/0xf0 [102097.050627] blk_mq_flush_plug_list+0x135/0x220 [102097.051534] blk_flush_plug_list+0xd7/0x100 [102097.052412] blk_finish_plug+0x25/0x36 [102097.053219] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102097.054363] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [102097.055370] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.056560] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.057702] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102097.058687] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102097.059603] ofd_commitrw+0x5b4/0xd20 [ofd] [102097.060420] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.061483] obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.062422] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102097.063399] ? flush_work+0x42/0x1d0 [102097.064216] ? internal_add_timer+0x42/0x70 [102097.065034] ? _cond_resched+0x15/0x30 [102097.065814] ? mutex_lock+0xe/0x30 [102097.066526] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102097.067579] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102097.068737] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102097.069878] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102097.070808] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102097.071821] kthread+0x134/0x150 [102097.072484] ? set_kthread_struct+0x50/0x50 [102097.073252] ret_from_fork+0x1f/0x40 [102097.073956] CPU: 7 PID: 527144 Comm: ll_ost_io01_040 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102097.074264] ll_ost_io02_028: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102097.076400] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102097.076401] ,cpuset=/ [102097.078107] Call Trace: [102097.079690] ,mems_allowed=0 [102097.080288] dump_stack+0x41/0x60 [102097.081547] warn_alloc.cold.127+0x7b/0x108 [102097.083552] __alloc_pages_slowpath+0xcb2/0xcd0 [102097.084412] __alloc_pages_nodemask+0x2e2/0x330 [102097.085272] kmalloc_order+0x28/0x90 [102097.086027] kmalloc_order_trace+0x1d/0xb0 [102097.086809] __kmalloc+0x203/0x250 [102097.087487] virtqueue_add+0x493/0xc70 [102097.088253] ? finish_wait+0x80/0x80 [102097.088989] virtqueue_add_sgs+0x80/0xa0 [102097.089758] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102097.090725] ? scsi_alloc_sgtables+0x84/0x1c0 [102097.091556] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102097.092485] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102097.093474] scsi_queue_rq+0x512/0xb10 [102097.094191] __blk_mq_try_issue_directly+0x163/0x200 [102097.095084] blk_mq_request_issue_directly+0x4e/0xb0 [102097.095966] blk_mq_try_issue_list_directly+0x62/0x100 [102097.096895] blk_mq_sched_insert_requests+0xa4/0xf0 [102097.097783] blk_mq_flush_plug_list+0x135/0x220 [102097.098638] blk_flush_plug_list+0xd7/0x100 [102097.099447] blk_finish_plug+0x25/0x36 [102097.100165] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102097.101229] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102097.102219] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.103339] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.104436] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102097.105399] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102097.106316] ofd_commitrw+0x5b4/0xd20 [ofd] [102097.107121] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.108085] obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.109026] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102097.109977] ? flush_work+0x42/0x1d0 [102097.110689] ? internal_add_timer+0x42/0x70 [102097.111475] ? _cond_resched+0x15/0x30 [102097.112196] ? mutex_lock+0xe/0x30 [102097.112861] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102097.113846] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102097.114952] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102097.116065] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102097.116978] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102097.117960] kthread+0x134/0x150 [102097.118633] ? set_kthread_struct+0x50/0x50 [102097.119473] ret_from_fork+0x1f/0x40 [102097.120196] CPU: 11 PID: 527185 Comm: ll_ost_io02_028 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102097.123149] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102097.124786] Call Trace: [102097.125366] dump_stack+0x41/0x60 [102097.126045] warn_alloc.cold.127+0x7b/0x108 [102097.126864] __alloc_pages_slowpath+0xcb2/0xcd0 [102097.127724] __alloc_pages_nodemask+0x2e2/0x330 [102097.128594] kmalloc_order+0x28/0x90 [102097.129320] kmalloc_order_trace+0x1d/0xb0 [102097.130116] __kmalloc+0x203/0x250 [102097.130824] virtqueue_add+0x493/0xc70 [102097.131607] ? finish_wait+0x80/0x80 [102097.132336] virtqueue_add_sgs+0x80/0xa0 [102097.133100] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102097.134077] ? scsi_alloc_sgtables+0x84/0x1c0 [102097.134924] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102097.135844] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102097.136868] scsi_queue_rq+0x512/0xb10 [102097.137643] __blk_mq_try_issue_directly+0x163/0x200 [102097.138891] blk_mq_request_issue_directly+0x4e/0xb0 [102097.140205] blk_mq_try_issue_list_directly+0x62/0x100 [102097.141378] blk_mq_sched_insert_requests+0xa4/0xf0 [102097.142312] blk_mq_flush_plug_list+0x135/0x220 [102097.143183] blk_flush_plug_list+0xd7/0x100 [102097.144017] blk_finish_plug+0x25/0x36 [102097.144942] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102097.146472] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [102097.147808] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.148980] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.150122] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102097.151111] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102097.152148] ofd_commitrw+0x5b4/0xd20 [ofd] [102097.153345] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.154467] obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.155457] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102097.156464] ? flush_work+0x42/0x1d0 [102097.157215] ? internal_add_timer+0x42/0x70 [102097.158025] ? _cond_resched+0x15/0x30 [102097.158765] ? mutex_lock+0xe/0x30 [102097.159459] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102097.160466] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102097.161598] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102097.162737] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102097.163687] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102097.164694] kthread+0x134/0x150 [102097.165387] ? set_kthread_struct+0x50/0x50 [102097.166214] ret_from_fork+0x1f/0x40 [102097.166936] CPU: 13 PID: 527445 Comm: ll_ost_io03_060 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102097.170018] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102097.172387] Call Trace: [102097.173313] dump_stack+0x41/0x60 [102097.174404] warn_alloc.cold.127+0x7b/0x108 [102097.175624] __alloc_pages_slowpath+0xcb2/0xcd0 [102097.180039] __alloc_pages_nodemask+0x2e2/0x330 [102097.181370] kmalloc_order+0x28/0x90 [102097.182473] kmalloc_order_trace+0x1d/0xb0 [102097.183725] __kmalloc+0x203/0x250 [102097.184818] virtqueue_add+0x493/0xc70 [102097.185940] ? finish_wait+0x80/0x80 [102097.187008] virtqueue_add_sgs+0x80/0xa0 [102097.188134] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102097.189515] ? scsi_alloc_sgtables+0x84/0x1c0 [102097.190606] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102097.191927] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102097.193398] scsi_queue_rq+0x512/0xb10 [102097.194461] __blk_mq_try_issue_directly+0x163/0x200 [102097.195813] blk_mq_request_issue_directly+0x4e/0xb0 [102097.197158] blk_mq_try_issue_list_directly+0x62/0x100 [102097.198524] blk_mq_sched_insert_requests+0xa4/0xf0 [102097.199806] blk_mq_flush_plug_list+0x135/0x220 [102097.200974] blk_flush_plug_list+0xd7/0x100 [102097.202008] blk_finish_plug+0x25/0x36 [102097.203036] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102097.204382] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102097.205774] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.207422] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.208995] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102097.210359] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102097.211625] ofd_commitrw+0x5b4/0xd20 [ofd] [102097.212775] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.214122] obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.215450] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102097.216843] ? flush_work+0x42/0x1d0 [102097.217912] ? internal_add_timer+0x42/0x70 [102097.219058] ? _cond_resched+0x15/0x30 [102097.220115] ? mutex_lock+0xe/0x30 [102097.221044] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102097.222472] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102097.224067] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102097.225783] ? finish_wait+0x80/0x80 [102097.226846] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102097.228188] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102097.229632] kthread+0x134/0x150 [102097.230567] ? set_kthread_struct+0x50/0x50 [102097.231679] ret_from_fork+0x1f/0x40 [102097.232678] CPU: 1 PID: 527424 Comm: ll_ost_io00_047 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102097.235589] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102097.237664] Call Trace: [102097.238330] dump_stack+0x41/0x60 [102097.239087] warn_alloc.cold.127+0x7b/0x108 [102097.240043] __alloc_pages_slowpath+0xcb2/0xcd0 [102097.241014] ? __alloc_pages_nodemask+0x166/0x330 [102097.242006] __alloc_pages_nodemask+0x2e2/0x330 [102097.242998] kmalloc_order+0x28/0x90 [102097.243814] kmalloc_order_trace+0x1d/0xb0 [102097.244703] __kmalloc+0x203/0x250 [102097.245462] ? __wake_up_common+0x7a/0x190 [102097.246362] virtqueue_add+0x493/0xc70 [102097.247220] ? finish_wait+0x80/0x80 [102097.248019] virtqueue_add_sgs+0x80/0xa0 [102097.248879] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102097.249998] ? scsi_alloc_sgtables+0x84/0x1c0 [102097.250945] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102097.252104] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102097.253300] scsi_queue_rq+0x512/0xb10 [102097.254156] __blk_mq_try_issue_directly+0x163/0x200 [102097.255256] blk_mq_request_issue_directly+0x4e/0xb0 [102097.256429] blk_mq_try_issue_list_directly+0x62/0x100 [102097.257495] blk_mq_sched_insert_requests+0xa4/0xf0 [102097.258552] blk_mq_flush_plug_list+0x135/0x220 [102097.259575] blk_flush_plug_list+0xd7/0x100 [102097.260516] blk_finish_plug+0x25/0x36 [102097.261384] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102097.262638] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102097.263766] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.265067] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.266346] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102097.267453] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102097.268496] ofd_commitrw+0x5b4/0xd20 [ofd] [102097.269456] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.270548] obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.271597] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102097.272698] ? kvm_set_wallclock+0x10/0x10 [102097.273634] ? sched_clock+0x5/0x10 [102097.274449] ? newidle_balance+0x1e2/0x3b0 [102097.275351] ? internal_add_timer+0x42/0x70 [102097.276265] ? _cond_resched+0x15/0x30 [102097.277109] ? mutex_lock+0xe/0x30 [102097.277895] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102097.279071] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102097.280367] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102097.281665] ? finish_wait+0x80/0x80 [102097.282481] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102097.283605] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102097.284845] kthread+0x134/0x150 [102097.285588] ? set_kthread_struct+0x50/0x50 [102097.286498] ret_from_fork+0x1f/0x40 [102097.287297] CPU: 14 PID: 527056 Comm: ll_ost_io03_017 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102097.290212] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102097.292321] Call Trace: [102097.293158] dump_stack+0x41/0x60 [102097.294149] warn_alloc.cold.127+0x7b/0x108 [102097.295282] __alloc_pages_slowpath+0xcb2/0xcd0 [102097.296490] __alloc_pages_nodemask+0x2e2/0x330 [102097.297665] kmalloc_order+0x28/0x90 [102097.298653] kmalloc_order_trace+0x1d/0xb0 [102097.299755] __kmalloc+0x203/0x250 [102097.300724] virtqueue_add+0x493/0xc70 [102097.301773] ? finish_wait+0x80/0x80 [102097.302800] virtqueue_add_sgs+0x80/0xa0 [102097.303849] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102097.305375] ? scsi_alloc_sgtables+0x84/0x1c0 [102097.306656] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102097.308045] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102097.309537] scsi_queue_rq+0x512/0xb10 [102097.310628] __blk_mq_try_issue_directly+0x163/0x200 [102097.312033] blk_mq_request_issue_directly+0x4e/0xb0 [102097.313417] blk_mq_try_issue_list_directly+0x62/0x100 [102097.314696] blk_mq_sched_insert_requests+0xa4/0xf0 [102097.315934] blk_mq_flush_plug_list+0x135/0x220 [102097.317092] blk_flush_plug_list+0xd7/0x100 [102097.318215] blk_finish_plug+0x25/0x36 [102097.319254] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102097.320722] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102097.322101] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.323777] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102097.325358] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102097.326756] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102097.328079] ofd_commitrw+0x5b4/0xd20 [ofd] [102097.329235] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.330613] obd_commitrw+0x1b6/0x370 [ptlrpc] [102097.331917] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102097.333247] ? flush_work+0x42/0x1d0 [102097.334012] ? internal_add_timer+0x42/0x70 [102097.334822] ? _cond_resched+0x15/0x30 [102097.335543] ? mutex_lock+0xe/0x30 [102097.336228] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102097.337234] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102097.338371] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102097.339483] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102097.340385] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102097.341364] kthread+0x134/0x150 [102097.342015] ? set_kthread_struct+0x50/0x50 [102097.342804] ret_from_fork+0x1f/0x40 [102102.772171] warn_alloc: 2363 callbacks suppressed [102102.772175] ll_ost_io04_012: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102102.772328] ll_ost_io04_023: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102102.773387] ,cpuset= [102102.775468] ,cpuset= [102102.777510] / [102102.778938] / [102102.778937] ,mems_allowed=0 [102102.779549] ,mems_allowed=0 [102102.780065] CPU: 17 PID: 527194 Comm: ll_ost_io04_012 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102102.781477] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102102.786108] Call Trace: [102102.786752] dump_stack+0x41/0x60 [102102.787521] warn_alloc.cold.127+0x7b/0x108 [102102.788451] __alloc_pages_slowpath+0xcb2/0xcd0 [102102.789463] ? __alloc_pages_nodemask+0x166/0x330 [102102.790509] __alloc_pages_nodemask+0x2e2/0x330 [102102.791499] kmalloc_order+0x28/0x90 [102102.792323] kmalloc_order_trace+0x1d/0xb0 [102102.793223] __kmalloc+0x203/0x250 [102102.794023] virtqueue_add+0x493/0xc70 [102102.794870] ? finish_wait+0x80/0x80 [102102.795677] virtqueue_add_sgs+0x80/0xa0 [102102.796542] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102102.797637] ? scsi_alloc_sgtables+0x84/0x1c0 [102102.798551] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102102.799587] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102102.800709] scsi_queue_rq+0x512/0xb10 [102102.801530] __blk_mq_try_issue_directly+0x163/0x200 [102102.802543] blk_mq_request_issue_directly+0x4e/0xb0 [102102.803561] blk_mq_try_issue_list_directly+0x62/0x100 [102102.804616] blk_mq_sched_insert_requests+0xa4/0xf0 [102102.805626] blk_mq_flush_plug_list+0x135/0x220 [102102.806598] blk_flush_plug_list+0xd7/0x100 [102102.807503] blk_finish_plug+0x25/0x36 [102102.808355] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102102.809598] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102102.810742] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102102.812051] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102102.813339] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102102.814459] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102102.815518] ofd_commitrw+0x5b4/0xd20 [ofd] [102102.816470] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102102.817578] obd_commitrw+0x1b6/0x370 [ptlrpc] [102102.818659] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102102.819787] ? flush_work+0x42/0x1d0 [102102.820625] ? internal_add_timer+0x42/0x70 [102102.821550] ? _cond_resched+0x15/0x30 [102102.822389] ? mutex_lock+0xe/0x30 [102102.823174] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102102.824333] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102102.825616] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102102.826900] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102102.827957] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102102.829112] kthread+0x134/0x150 [102102.829870] ? set_kthread_struct+0x50/0x50 [102102.830813] ret_from_fork+0x1f/0x40 [102102.831642] warn_alloc_show_mem: 9 callbacks suppressed [102102.831644] Mem-Info: [102102.831644] CPU: 16 PID: 527238 Comm: ll_ost_io04_023 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102102.832723] active_anon:27844 inactive_anon:126243 isolated_anon:0 active_file:23538247 inactive_file:9004210 isolated_file:2 unevictable:0 dirty:77802 writeback:12 slab_reclaimable:1033363 slab_unreclaimable:669474 mapped:17826 shmem:145 pagetables:5835 bounce:0 free:3612784 free_pcp:436 free_cma:0 [102102.833356] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102102.835759] Node 0 active_anon:111376kB inactive_anon:504972kB active_file:94152988kB inactive_file:36016840kB unevictable:0kB isolated(anon):0kB isolated(file):8kB mapped:71304kB dirty:311208kB writeback:48kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:35664kB pagetables:23340kB all_unreclaimable? no [102102.842595] Call Trace: [102102.844355] Node 0 [102102.850211] dump_stack+0x41/0x60 [102102.850916] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102102.851566] warn_alloc.cold.127+0x7b/0x108 [102102.852399] lowmem_reserve[]: [102102.856611] __alloc_pages_slowpath+0xcb2/0xcd0 [102102.857594] 0 [102102.858389] ? __alloc_pages_nodemask+0x166/0x330 [102102.859458] 913 [102102.860048] __alloc_pages_nodemask+0x2e2/0x330 [102102.861140] 150007 [102102.861762] kmalloc_order+0x28/0x90 [102102.862821] 150007 [102102.863492] kmalloc_order_trace+0x1d/0xb0 [102102.864410] 150007 [102102.865102] __kmalloc+0x203/0x250 [102102.866768] virtqueue_add+0x493/0xc70 [102102.867642] Node 0 [102102.868201] virtqueue_add_sgs+0x80/0xa0 [102102.869099] DMA32 free:596456kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [102102.869750] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102102.870675] lowmem_reserve[]: [102102.875236] ? scsi_alloc_sgtables+0x84/0x1c0 [102102.876414] 0 [102102.877201] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102102.878213] 0 [102102.878778] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102102.879892] 149093 [102102.880455] scsi_queue_rq+0x512/0xb10 [102102.881669] 149093 [102102.882303] __blk_mq_try_issue_directly+0x163/0x200 [102102.883200] 149093 [102102.883839] blk_mq_request_issue_directly+0x4e/0xb0 [102102.885582] blk_mq_try_issue_list_directly+0x62/0x100 [102102.886690] Node 0 [102102.887225] blk_mq_sched_insert_requests+0xa4/0xf0 [102102.888355] Normal free:13824096kB min:49268kB low:201940kB high:354612kB active_anon:110996kB inactive_anon:504988kB active_file:94166924kB inactive_file:36019148kB unevictable:0kB writepending:311256kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1700kB local_pcp:124kB free_cma:0kB [102102.888980] blk_mq_flush_plug_list+0x135/0x220 [102102.890057] lowmem_reserve[]: [102102.895071] blk_flush_plug_list+0xd7/0x100 [102102.896088] 0 [102102.896860] blk_finish_plug+0x25/0x36 [102102.897822] 0 [102102.898371] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102102.899252] 0 [102102.899795] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102102.901023] 0 [102102.901579] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102102.902701] 0 [102102.903265] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102102.905127] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102102.906405] Node 0 [102102.906922] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102102.908042] DMA: [102102.908653] ofd_commitrw+0x5b4/0xd20 [ofd] [102102.909696] 0*4kB [102102.910276] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102102.911227] 0*8kB [102102.911817] obd_commitrw+0x1b6/0x370 [ptlrpc] [102102.912857] 0*16kB [102102.913443] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102102.914440] 0*32kB [102102.915045] ? flush_work+0x42/0x1d0 [102102.916063] 0*64kB [102102.916665] ? internal_add_timer+0x42/0x70 [102102.917499] 0*128kB [102102.918102] ? _cond_resched+0x15/0x30 [102102.919053] 0*256kB [102102.919670] ? mutex_lock+0xe/0x30 [102102.920544] 0*512kB [102102.921157] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102102.921977] 1*1024kB [102102.922589] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102102.923680] (U) [102102.924304] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102102.925506] 1*2048kB [102102.926058] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102102.927278] (M) [102102.927910] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102102.928899] 2*4096kB [102102.929455] kthread+0x134/0x150 [102102.930517] (M) [102102.931146] ? set_kthread_struct+0x50/0x50 [102102.931913] = 11264kB [102102.932464] ret_from_fork+0x1f/0x40 [102102.933393] Node 0 DMA32: 16*4kB (UMH) 19*8kB (UMH) 63*16kB (UMEH) 38*32kB (UMEH) 36*64kB (UM) 40*128kB (UMH) 39*256kB (UMH) 38*512kB (UME) 32*1024kB (UMH) 36*2048kB (UM) 110*4096kB (UM) = 596360kB [102102.939350] Node 0 Normal: 1976742*4kB (UMEH) 671595*8kB (UMEH) 33519*16kB (UMEH) 7*32kB (H) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 13816256kB [102102.942846] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [102102.944877] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [102102.946774] 32548889 total pagecache pages [102102.947927] 1617 pages in swap cache [102102.948956] Swap cache stats: add 1810819, delete 1809219, find 108117/131031 [102102.950628] Free swap = 4727064kB [102102.951663] Total swap = 11075580kB [102102.952613] 39321462 pages RAM [102102.953470] 0 pages HighMem/MovableOnly [102102.954443] 905848 pages reserved [102102.955382] 0 pages hwpoisoned [102103.203371] ll_ost_io00_004: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [102103.205652] CPU: 3 PID: 527121 Comm: ll_ost_io00_004 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.207778] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.209455] Call Trace: [102103.210131] dump_stack+0x41/0x60 [102103.210819] warn_alloc.cold.127+0x7b/0x108 [102103.211656] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.212514] ? __alloc_pages_nodemask+0x166/0x330 [102103.213395] __alloc_pages_nodemask+0x2e2/0x330 [102103.214238] kmalloc_order+0x28/0x90 [102103.214957] kmalloc_order_trace+0x1d/0xb0 [102103.215742] __kmalloc+0x203/0x250 [102103.216453] virtqueue_add+0x493/0xc70 [102103.217204] ? finish_wait+0x80/0x80 [102103.217968] virtqueue_add_sgs+0x80/0xa0 [102103.218752] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.219728] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.220547] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.221470] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.222481] scsi_queue_rq+0x512/0xb10 [102103.223213] __blk_mq_try_issue_directly+0x163/0x200 [102103.224126] blk_mq_request_issue_directly+0x4e/0xb0 [102103.225067] blk_mq_try_issue_list_directly+0x62/0x100 [102103.226035] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.226991] blk_mq_flush_plug_list+0x135/0x220 [102103.227936] blk_flush_plug_list+0xd7/0x100 [102103.228784] blk_finish_plug+0x25/0x36 [102103.229758] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.231284] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102103.232395] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.233660] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.235022] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.236072] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.237119] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.237971] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.239035] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.239987] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.240985] ? flush_work+0x42/0x1d0 [102103.241709] ? internal_add_timer+0x42/0x70 [102103.242499] ? _cond_resched+0x15/0x30 [102103.243221] ? mutex_lock+0xe/0x30 [102103.243899] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.244900] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.246030] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.247288] ? finish_wait+0x80/0x80 [102103.248200] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.249216] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.250194] kthread+0x134/0x150 [102103.250842] ? set_kthread_struct+0x50/0x50 [102103.251636] ret_from_fork+0x1f/0x40 [102103.284353] ll_ost_io00_063: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102103.284355] ll_ost_io04_041: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102103.284388] ll_ost_io01_022: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102103.284389] ll_ost_io01_065: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102103.284390] ll_ost_io01_018: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102103.284390] ll_ost_io01_015: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102103.284393] ,cpuset= [102103.284394] ,cpuset= [102103.284394] ,cpuset= [102103.284394] ,cpuset= [102103.284396] / [102103.284397] / [102103.284397] ,mems_allowed=0 [102103.284398] ,mems_allowed=0 [102103.284399] / [102103.284400] ,mems_allowed=0 [102103.284400] CPU: 4 PID: 527068 Comm: ll_ost_io01_015 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.284401] / [102103.284403] ,mems_allowed=0 [102103.284403] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.284405] Call Trace: [102103.284409] dump_stack+0x41/0x60 [102103.284416] warn_alloc.cold.127+0x7b/0x108 [102103.284422] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.284425] ? __alloc_pages_nodemask+0x166/0x330 [102103.284428] __alloc_pages_nodemask+0x2e2/0x330 [102103.284431] kmalloc_order+0x28/0x90 [102103.284437] kmalloc_order_trace+0x1d/0xb0 [102103.284440] __kmalloc+0x203/0x250 [102103.284445] virtqueue_add+0x493/0xc70 [102103.284451] ? finish_wait+0x80/0x80 [102103.284455] virtqueue_add_sgs+0x80/0xa0 [102103.284460] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.284465] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.284470] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.284473] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.284476] scsi_queue_rq+0x512/0xb10 [102103.284480] __blk_mq_try_issue_directly+0x163/0x200 [102103.284485] blk_mq_request_issue_directly+0x4e/0xb0 [102103.284489] blk_mq_try_issue_list_directly+0x62/0x100 [102103.284493] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.284496] blk_mq_flush_plug_list+0x135/0x220 [102103.284500] blk_flush_plug_list+0xd7/0x100 [102103.284504] blk_finish_plug+0x25/0x36 [102103.284509] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.284542] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102103.284570] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.284595] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.284621] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.284646] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.284664] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.284682] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.284791] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.284889] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.284991] ? flush_work+0x42/0x1d0 [102103.284996] ? internal_add_timer+0x42/0x70 [102103.284999] ? _cond_resched+0x15/0x30 [102103.285001] ? mutex_lock+0xe/0x30 [102103.285004] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.285109] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.285213] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.285318] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.285427] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.285534] kthread+0x134/0x150 [102103.285537] ? set_kthread_struct+0x50/0x50 [102103.285540] ret_from_fork+0x1f/0x40 [102103.285546] CPU: 5 PID: 527079 Comm: ll_ost_io01_022 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.285549] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.285550] Call Trace: [102103.285552] dump_stack+0x41/0x60 [102103.285557] warn_alloc.cold.127+0x7b/0x108 [102103.285562] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.285565] ? __alloc_pages_nodemask+0x166/0x330 [102103.285568] __alloc_pages_nodemask+0x2e2/0x330 [102103.285571] kmalloc_order+0x28/0x90 [102103.285575] kmalloc_order_trace+0x1d/0xb0 [102103.285579] __kmalloc+0x203/0x250 [102103.285584] virtqueue_add+0x493/0xc70 [102103.285589] ? finish_wait+0x80/0x80 [102103.285593] virtqueue_add_sgs+0x80/0xa0 [102103.285597] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.285601] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.285606] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.285610] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.285613] scsi_queue_rq+0x512/0xb10 [102103.285616] __blk_mq_try_issue_directly+0x163/0x200 [102103.285622] blk_mq_request_issue_directly+0x4e/0xb0 [102103.285626] blk_mq_try_issue_list_directly+0x62/0x100 [102103.285629] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.285634] blk_mq_flush_plug_list+0x135/0x220 [102103.285637] blk_flush_plug_list+0xd7/0x100 [102103.285642] blk_finish_plug+0x25/0x36 [102103.285645] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.285670] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102103.285692] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.285710] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.285728] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.285746] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.285760] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.285771] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.285856] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.285925] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.285991] ? flush_work+0x42/0x1d0 [102103.285995] ? internal_add_timer+0x42/0x70 [102103.285997] ? _cond_resched+0x15/0x30 [102103.285999] ? mutex_lock+0xe/0x30 [102103.286001] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.286077] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.286166] ,cpuset= [102103.286148] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.286210] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.286272] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.286348] kthread+0x134/0x150 [102103.286351] ? set_kthread_struct+0x50/0x50 [102103.286354] ret_from_fork+0x1f/0x40 [102103.286358] CPU: 6 PID: 527203 Comm: ll_ost_io01_065 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.286362] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.286363] Call Trace: [102103.286366] dump_stack+0x41/0x60 [102103.286372] warn_alloc.cold.127+0x7b/0x108 [102103.286377] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.286381] ? __alloc_pages_nodemask+0x166/0x330 [102103.286384] __alloc_pages_nodemask+0x2e2/0x330 [102103.286386] kmalloc_order+0x28/0x90 [102103.286390] kmalloc_order_trace+0x1d/0xb0 [102103.286394] __kmalloc+0x203/0x250 [102103.286398] virtqueue_add+0x493/0xc70 [102103.286402] ? finish_wait+0x80/0x80 [102103.286406] virtqueue_add_sgs+0x80/0xa0 [102103.286410] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.286415] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.286419] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.286422] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.286426] scsi_queue_rq+0x512/0xb10 [102103.286428] __blk_mq_try_issue_directly+0x163/0x200 [102103.286433] blk_mq_request_issue_directly+0x4e/0xb0 [102103.286437] blk_mq_try_issue_list_directly+0x62/0x100 [102103.286440] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.286444] blk_mq_flush_plug_list+0x135/0x220 [102103.286448] blk_flush_plug_list+0xd7/0x100 [102103.286452] blk_finish_plug+0x25/0x36 [102103.286456] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.286486] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102103.286512] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.286538] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.286564] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.286589] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.286606] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.286622] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.286729] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.286831] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.286930] ? flush_work+0x42/0x1d0 [102103.286935] ? internal_add_timer+0x42/0x70 [102103.286937] ? _cond_resched+0x15/0x30 [102103.286939] ? mutex_lock+0xe/0x30 [102103.286942] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.287043] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.287142] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.287237] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.287340] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.287447] kthread+0x134/0x150 [102103.287450] ? set_kthread_struct+0x50/0x50 [102103.287453] ret_from_fork+0x1f/0x40 [102103.287458] CPU: 7 PID: 527075 Comm: ll_ost_io01_018 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.287462] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.287463] Call Trace: [102103.287465] dump_stack+0x41/0x60 [102103.287470] warn_alloc.cold.127+0x7b/0x108 [102103.287475] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.287478] ? __alloc_pages_nodemask+0x166/0x330 [102103.287481] __alloc_pages_nodemask+0x2e2/0x330 [102103.287484] kmalloc_order+0x28/0x90 [102103.287489] kmalloc_order_trace+0x1d/0xb0 [102103.287492] __kmalloc+0x203/0x250 [102103.287497] virtqueue_add+0x493/0xc70 [102103.287501] ? finish_wait+0x80/0x80 [102103.287505] virtqueue_add_sgs+0x80/0xa0 [102103.287509] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.287513] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.287518] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.287522] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.287526] scsi_queue_rq+0x512/0xb10 [102103.287529] __blk_mq_try_issue_directly+0x163/0x200 [102103.287534] blk_mq_request_issue_directly+0x4e/0xb0 [102103.287538] blk_mq_try_issue_list_directly+0x62/0x100 [102103.287541] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.287543] blk_mq_flush_plug_list+0x135/0x220 [102103.287547] blk_flush_plug_list+0xd7/0x100 [102103.287550] blk_finish_plug+0x25/0x36 [102103.287554] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.287575] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102103.287591] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.287609] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.287628] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.287646] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.287657] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.287668] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.287740] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.287822] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.287888] ? flush_work+0x42/0x1d0 [102103.287892] ? internal_add_timer+0x42/0x70 [102103.287894] ? _cond_resched+0x15/0x30 [102103.287896] ? mutex_lock+0xe/0x30 [102103.287898] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.287965] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.288047] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.288129] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.288198] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.288265] kthread+0x134/0x150 [102103.288268] ? set_kthread_struct+0x50/0x50 [102103.288271] ret_from_fork+0x1f/0x40 [102103.288678] ,cpuset= [102103.290592] / [102103.371388] ll_ost_io00_032: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null) [102103.372184] ,mems_allowed=0 [102103.372184] / [102103.372189] CPU: 2 PID: 527756 Comm: ll_ost_io00_063 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.372192] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.372194] Call Trace: [102103.372199] dump_stack+0x41/0x60 [102103.372207] warn_alloc.cold.127+0x7b/0x108 [102103.372214] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.372218] ? __alloc_pages_nodemask+0x166/0x330 [102103.372221] __alloc_pages_nodemask+0x2e2/0x330 [102103.372224] kmalloc_order+0x28/0x90 [102103.372230] kmalloc_order_trace+0x1d/0xb0 [102103.372234] __kmalloc+0x203/0x250 [102103.372240] virtqueue_add+0x493/0xc70 [102103.372247] ? finish_wait+0x80/0x80 [102103.372251] virtqueue_add_sgs+0x80/0xa0 [102103.372256] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.372261] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.372267] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.372271] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.372275] scsi_queue_rq+0x512/0xb10 [102103.372278] __blk_mq_try_issue_directly+0x163/0x200 [102103.372284] blk_mq_request_issue_directly+0x4e/0xb0 [102103.372289] blk_mq_try_issue_list_directly+0x62/0x100 [102103.372293] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.372296] blk_mq_flush_plug_list+0x135/0x220 [102103.372300] blk_flush_plug_list+0xd7/0x100 [102103.372305] blk_finish_plug+0x25/0x36 [102103.372311] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.372354] ? ldiskfs_map_blocks+0x41b/0x610 [ldiskfs] [102103.372387] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.372415] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.372444] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.372474] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.372495] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.372514] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.372650] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.372768] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.372883] ? flush_work+0x42/0x1d0 [102103.372889] ? internal_add_timer+0x42/0x70 [102103.372893] ? _cond_resched+0x15/0x30 [102103.372896] ? mutex_lock+0xe/0x30 [102103.372899] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.373012] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.373123] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.373231] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.373288] ,cpuset= [102103.373338] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.373438] kthread+0x134/0x150 [102103.373443] ? set_kthread_struct+0x50/0x50 [102103.373446] ret_from_fork+0x1f/0x40 [102103.374832] ,mems_allowed=0 [102103.376356] / [102103.379130] ,mems_allowed=0 [102103.380084] CPU: 18 PID: 527678 Comm: ll_ost_io04_041 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.382201] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.537993] Call Trace: [102103.538529] dump_stack+0x41/0x60 [102103.539182] warn_alloc.cold.127+0x7b/0x108 [102103.539977] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.540804] ? __alloc_pages_nodemask+0x166/0x330 [102103.541664] __alloc_pages_nodemask+0x2e2/0x330 [102103.542476] kmalloc_order+0x28/0x90 [102103.543161] kmalloc_order_trace+0x1d/0xb0 [102103.543946] __kmalloc+0x203/0x250 [102103.544652] ? __wake_up_common+0x7a/0x190 [102103.545625] virtqueue_add+0x493/0xc70 [102103.546626] ? finish_wait+0x80/0x80 [102103.547582] virtqueue_add_sgs+0x80/0xa0 [102103.548695] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.549872] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.550770] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.551800] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.552913] scsi_queue_rq+0x512/0xb10 [102103.553747] __blk_mq_try_issue_directly+0x163/0x200 [102103.554767] blk_mq_request_issue_directly+0x4e/0xb0 [102103.555829] blk_mq_try_issue_list_directly+0x62/0x100 [102103.556765] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.557645] blk_mq_flush_plug_list+0x135/0x220 [102103.558479] blk_flush_plug_list+0xd7/0x100 [102103.559340] blk_finish_plug+0x25/0x36 [102103.560324] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.561702] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102103.563042] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.564510] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.565634] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.566603] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.567499] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.568280] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.569261] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.570359] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.571270] ? flush_work+0x42/0x1d0 [102103.571975] ? internal_add_timer+0x42/0x70 [102103.572745] ? _cond_resched+0x15/0x30 [102103.573457] ? mutex_lock+0xe/0x30 [102103.574111] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.575074] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.576136] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.577197] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.578089] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.579073] kthread+0x134/0x150 [102103.579785] ? set_kthread_struct+0x50/0x50 [102103.580645] ret_from_fork+0x1f/0x40 [102103.581363] CPU: 3 PID: 527264 Comm: ll_ost_io00_032 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [102103.584246] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [102103.586416] Call Trace: [102103.587293] dump_stack+0x41/0x60 [102103.588298] warn_alloc.cold.127+0x7b/0x108 [102103.589420] __alloc_pages_slowpath+0xcb2/0xcd0 [102103.590619] __alloc_pages_nodemask+0x2e2/0x330 [102103.591837] kmalloc_order+0x28/0x90 [102103.592901] kmalloc_order_trace+0x1d/0xb0 [102103.594032] __kmalloc+0x203/0x250 [102103.595064] virtqueue_add+0x493/0xc70 [102103.596174] ? finish_wait+0x80/0x80 [102103.597264] virtqueue_add_sgs+0x80/0xa0 [102103.598428] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [102103.599914] ? scsi_alloc_sgtables+0x84/0x1c0 [102103.601237] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [102103.602806] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [102103.604354] scsi_queue_rq+0x512/0xb10 [102103.605522] __blk_mq_try_issue_directly+0x163/0x200 [102103.606828] blk_mq_request_issue_directly+0x4e/0xb0 [102103.608097] blk_mq_try_issue_list_directly+0x62/0x100 [102103.609371] blk_mq_sched_insert_requests+0xa4/0xf0 [102103.610662] blk_mq_flush_plug_list+0x135/0x220 [102103.611822] blk_flush_plug_list+0xd7/0x100 [102103.612952] blk_finish_plug+0x25/0x36 [102103.613988] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [102103.615439] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [102103.616782] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.618361] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [102103.619885] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [102103.621256] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [102103.622491] ofd_commitrw+0x5b4/0xd20 [ofd] [102103.623650] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.625047] obd_commitrw+0x1b6/0x370 [ptlrpc] [102103.626341] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [102103.627704] ? flush_work+0x42/0x1d0 [102103.628705] ? internal_add_timer+0x42/0x70 [102103.629792] ? _cond_resched+0x15/0x30 [102103.630785] ? mutex_lock+0xe/0x30 [102103.631718] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [102103.633097] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [102103.634644] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [102103.636158] ptlrpc_main+0xbec/0x1530 [ptlrpc] [102103.637408] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [102103.638776] kthread+0x134/0x150 [102103.639730] ? set_kthread_struct+0x50/0x50 [102103.640871] ret_from_fork+0x1f/0x40 [103494.520584] warn_alloc: 4945 callbacks suppressed [103494.520588] ll_ost_io03_046: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [103494.524951] CPU: 15 PID: 527323 Comm: ll_ost_io03_046 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [103494.527502] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [103494.529009] Call Trace: [103494.529590] dump_stack+0x41/0x60 [103494.530349] warn_alloc.cold.127+0x7b/0x108 [103494.531452] __alloc_pages_slowpath+0xcb2/0xcd0 [103494.532669] ? __alloc_pages_nodemask+0x166/0x330 [103494.533919] __alloc_pages_nodemask+0x2e2/0x330 [103494.535142] kmalloc_order+0x28/0x90 [103494.536181] kmalloc_order_trace+0x1d/0xb0 [103494.537304] __kmalloc+0x203/0x250 [103494.538295] ? __wake_up_common+0x7a/0x190 [103494.539411] virtqueue_add+0x493/0xc70 [103494.540434] ? finish_wait+0x80/0x80 [103494.541415] virtqueue_add_sgs+0x80/0xa0 [103494.542487] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [103494.543891] ? scsi_alloc_sgtables+0x84/0x1c0 [103494.545082] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [103494.546433] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [103494.547899] scsi_queue_rq+0x512/0xb10 [103494.548975] __blk_mq_try_issue_directly+0x163/0x200 [103494.550296] blk_mq_request_issue_directly+0x4e/0xb0 [103494.551617] blk_mq_try_issue_list_directly+0x62/0x100 [103494.552929] blk_mq_sched_insert_requests+0xa4/0xf0 [103494.554182] blk_mq_flush_plug_list+0x135/0x220 [103494.555378] blk_flush_plug_list+0xd7/0x100 [103494.556501] blk_finish_plug+0x25/0x36 [103494.557612] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [103494.559155] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [103494.560626] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [103494.562222] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [103494.563793] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [103494.565155] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [103494.566509] ofd_commitrw+0x5b4/0xd20 [ofd] [103494.567639] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [103494.568312] ll_ost_io03_049: page allocation failure: order:3, mode:0x484020(GFP_ATOMIC|__GFP_COMP), nodemask=(null),cpuset=/,mems_allowed=0 [103494.568971] obd_commitrw+0x1b6/0x370 [ptlrpc] [103494.573015] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [103494.574496] ? flush_work+0x42/0x1d0 [103494.575567] ? internal_add_timer+0x42/0x70 [103494.576764] ? _cond_resched+0x15/0x30 [103494.577886] ? mutex_lock+0xe/0x30 [103494.578924] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [103494.580373] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [103494.581948] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [103494.583534] ptlrpc_main+0xbec/0x1530 [ptlrpc] [103494.584849] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [103494.586306] kthread+0x134/0x150 [103494.587276] ? set_kthread_struct+0x50/0x50 [103494.588431] ret_from_fork+0x1f/0x40 [103494.589452] warn_alloc_show_mem: 9 callbacks suppressed [103494.589454] CPU: 12 PID: 527326 Comm: ll_ost_io03_049 Kdump: loaded Tainted: G OE -------- - - 4.18.0-553.27.1.el8_lustre.x86_64 #1 [103494.589461] Mem-Info: [103494.590814] Hardware name: DDN SFA18KXE, BIOS rel-1.16.2-0-gea1b7a073390-prebuilt.qemu.org 04/01/2014 [103494.593745] active_anon:26628 inactive_anon:79658 isolated_anon:0 active_file:20525660 inactive_file:15332357 isolated_file:1 unevictable:0 dirty:1077 writeback:0 slab_reclaimable:1037863 slab_unreclaimable:649098 mapped:18143 shmem:145 pagetables:5906 bounce:0 free:355951 free_pcp:504 free_cma:0 [103494.594550] Call Trace: [103494.596747] Node 0 active_anon:106512kB inactive_anon:318632kB active_file:82104092kB inactive_file:61329428kB unevictable:0kB isolated(anon):0kB isolated(file):4kB mapped:72572kB dirty:4308kB writeback:0kB shmem:580kB shmem_thp: 0kB shmem_pmdmapped: 0kB anon_thp: 0kB writeback_tmp:0kB kernel_stack:36384kB pagetables:23624kB all_unreclaimable? no [103494.604648] dump_stack+0x41/0x60 [103494.605501] Node 0 [103494.611172] warn_alloc.cold.127+0x7b/0x108 [103494.612174] DMA free:11264kB min:4kB low:16kB high:28kB active_anon:0kB inactive_anon:0kB active_file:0kB inactive_file:0kB unevictable:0kB writepending:0kB present:15992kB managed:15360kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [103494.612954] __alloc_pages_slowpath+0xcb2/0xcd0 [103494.614138] lowmem_reserve[]: [103494.618870] ? __alloc_pages_nodemask+0x166/0x330 [103494.620186] 0 [103494.621005] __alloc_pages_nodemask+0x2e2/0x330 [103494.622224] 913 [103494.622937] kmalloc_order+0x28/0x90 [103494.624247] 150007 [103494.624881] kmalloc_order_trace+0x1d/0xb0 [103494.626008] 150007 [103494.626623] __kmalloc+0x203/0x250 [103494.627813] 150007 [103494.628630] virtqueue_add+0x493/0xc70 [103494.630530] ? finish_wait+0x80/0x80 [103494.631671] Node 0 [103494.632312] virtqueue_add_sgs+0x80/0xa0 [103494.633363] DMA32 free:596392kB min:300kB low:1232kB high:2164kB active_anon:380kB inactive_anon:4kB active_file:12kB inactive_file:100kB unevictable:0kB writepending:0kB present:2080608kB managed:966496kB mlocked:0kB bounce:0kB free_pcp:0kB local_pcp:0kB free_cma:0kB [103494.634050] __virtscsi_add_cmd+0x148/0x270 [virtio_scsi] [103494.635242] lowmem_reserve[]: 0 [103494.639370] ? scsi_alloc_sgtables+0x84/0x1c0 [103494.640900] 0 [103494.641787] virtscsi_add_cmd+0x38/0xa0 [virtio_scsi] [103494.643086] 149093 [103494.643847] virtscsi_queuecommand+0x186/0x2d0 [virtio_scsi] [103494.645268] 149093 [103494.646125] scsi_queue_rq+0x512/0xb10 [103494.647693] 149093 [103494.648548] __blk_mq_try_issue_directly+0x163/0x200 [103494.650565] blk_mq_request_issue_directly+0x4e/0xb0 [103494.651855] Node 0 [103494.652620] blk_mq_try_issue_list_directly+0x62/0x100 [103494.653645] Normal free:791628kB min:49268kB low:201940kB high:354612kB active_anon:106132kB inactive_anon:318612kB active_file:82124008kB inactive_file:61333432kB unevictable:0kB writepending:4308kB present:155189248kB managed:152680600kB mlocked:0kB bounce:0kB free_pcp:1968kB local_pcp:4kB free_cma:0kB [103494.654481] blk_mq_sched_insert_requests+0xa4/0xf0 [103494.655525] lowmem_reserve[]: [103494.661555] blk_mq_flush_plug_list+0x135/0x220 [103494.662784] 0 [103494.663652] blk_flush_plug_list+0xd7/0x100 [103494.664946] 0 [103494.665661] blk_finish_plug+0x25/0x36 [103494.666908] 0 [103494.667536] osd_do_bio.constprop.49+0xd86/0xeb0 [osd_ldiskfs] [103494.668645] 0 [103494.669357] ? ldiskfs_map_blocks+0x607/0x610 [ldiskfs] [103494.670845] 0 [103494.671568] ? osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [103494.673707] osd_ldiskfs_map_inode_pages+0x8c0/0x930 [osd_ldiskfs] [103494.675277] Node 0 [103494.675954] osd_write_commit+0x5e2/0x990 [osd_ldiskfs] [103494.677481] DMA: [103494.678198] ofd_commitrw_write+0x77e/0x1ad0 [ofd] [103494.679523] 0*4kB [103494.680255] ofd_commitrw+0x5b4/0xd20 [ofd] [103494.681504] 0*8kB [103494.682154] ? obd_commitrw+0x1b6/0x370 [ptlrpc] [103494.683263] 0*16kB [103494.684077] obd_commitrw+0x1b6/0x370 [ptlrpc] [103494.685340] 0*32kB [103494.686141] tgt_brw_write+0x1374/0x1cb0 [ptlrpc] [103494.687254] 0*64kB [103494.688000] ? newidle_balance+0x2b6/0x3b0 [103494.689268] 0*128kB [103494.690085] ? internal_add_timer+0x42/0x70 [103494.691203] 0*256kB [103494.691952] ? _cond_resched+0x15/0x30 [103494.693140] 0*512kB [103494.693836] ? mutex_lock+0xe/0x30 [103494.694927] 1*1024kB [103494.695689] tgt_request_handle+0xccd/0x1a20 [ptlrpc] [103494.696614] (U) [103494.697398] ? ptlrpc_nrs_req_get_nolock0+0xff/0x1f0 [ptlrpc] [103494.698750] 1*2048kB [103494.699398] ptlrpc_server_handle_request+0x323/0xbe0 [ptlrpc] [103494.700870] (M) [103494.701709] ? finish_wait+0x80/0x80 [103494.703192] 2*4096kB [103494.703760] ptlrpc_main+0xbec/0x1530 [ptlrpc] [103494.704514] (M) [103494.705099] ? ptlrpc_wait_event+0x590/0x590 [ptlrpc] [103494.705966] = 11264kB [103494.706497] kthread+0x134/0x150 [103494.707457] Node 0 [103494.708035] ? set_kthread_struct+0x50/0x50 [103494.708730] DMA32: [103494.709417] ret_from_fork+0x1f/0x40 [103494.710522] 34*4kB (UM) 36*8kB (UMH) 84*16kB (UMEH) 35*32kB (MEH) 26*64kB (UMH) 36*128kB (UMH) 36*256kB (UM) 35*512kB (UMEH) 31*1024kB (UM) 34*2048kB (UM) 112*4096kB (UM) = 596424kB [103494.716318] Node 0 Normal: 146273*4kB (UME) 21305*8kB (UME) 2810*16kB (UMEH) 1*32kB (H) 0*64kB 0*128kB 0*256kB 0*512kB 0*1024kB 0*2048kB 0*4096kB = 800524kB [103494.719426] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=1048576kB [103494.721471] Node 0 hugepages_total=0 hugepages_free=0 hugepages_surp=0 hugepages_size=2048kB [103494.723325] 35857993 total pagecache pages [103494.724499] 1252 pages in swap cache [103494.725624] Swap cache stats: add 1899282, delete 1898069, find 114983/139707 [103494.727326] Free swap = 4434568kB [103494.728247] Total swap = 11075580kB [103494.729078] 39321462 pages RAM [103494.729880] 0 pages HighMem/MovableOnly [103494.731082] 905848 pages reserved [103494.732085] 0 pages hwpoisoned [126130.353319] mlx5_core 0000:03:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 1 8000000000000000 [126130.367840] mlx5_core 0000:04:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 1 8000000000000000 [126274.777416] LustreError: 185190:0:(ldlm_lockd.c:261:expired_lock_main()) ### lock callback timer expired after 82s: evicting client at 172.25.80.23@tcp ns: mdt-sfa18k03-MDT0001_UUID lock: 00000000fb44c62b/0xcf4e9953b3f4d8c1 lrc: 3/0,0 mode: PW/PW res: [0x28003bd3e:0x19a11:0x0].0x0 bits 0x41/0x0 rrc: 6 type: IBT gid 0 flags: 0x60200400000020 nid: 172.25.80.23@tcp remote: 0x5876d76d8d3e86a3 expref: 2566 pid: 531728 timeout: 126263 lvb_type: 0 [126274.787334] LustreError: 185190:0:(client.c:1256:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@00000000ef2e8167 x1816365546510848/t0(0) o104->sfa18k03-MDT0001@172.25.80.23@tcp:15/16 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' [126274.880371] LustreError: 188922:0:(ldlm_lockd.c:1427:ldlm_handle_enqueue0()) ### lock on destroyed export 0000000066f91dd7 ns: mdt-sfa18k03-MDT0001_UUID lock: 000000005b4e9572/0xcf4e9953b3f50656 lrc: 3/0,0 mode: PR/PR res: [0x28003bd3e:0x19a11:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x50200000000000 nid: 172.25.80.23@tcp remote: 0x5876d76d8d3ebf13 expref: 856 pid: 188922 timeout: 0 lvb_type: 0 [126323.930305] LustreError: 185190:0:(ldlm_lockd.c:261:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 172.25.80.28@tcp ns: mdt-sfa18k03-MDT0001_UUID lock: 000000006e9b5f1b/0xcf4e9953b3fd7c7b lrc: 3/0,0 mode: PW/PW res: [0x28003bd3f:0x7a78:0x0].0x0 bits 0x41/0x0 rrc: 8 type: IBT gid 0 flags: 0x60200400000020 nid: 172.25.80.28@tcp remote: 0x1f5d523855665339 expref: 4763 pid: 527116 timeout: 126321 lvb_type: 0 [126324.117747] LustreError: 527407:0:(ldlm_lockd.c:1427:ldlm_handle_enqueue0()) ### lock on destroyed export 000000001f92d0d3 ns: mdt-sfa18k03-MDT0001_UUID lock: 00000000a8543ccb/0xcf4e9953b3fd9bd1 lrc: 3/0,0 mode: PR/PR res: [0x28003bd3f:0x7a78:0x0].0x0 bits 0x20/0x0 rrc: 3 type: IBT gid 0 flags: 0x50200000000000 nid: 172.25.80.28@tcp remote: 0x1f5d523855665c85 expref: 1140 pid: 527407 timeout: 0 lvb_type: 0 [126991.384045] mlx5_core 0000:03:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 0 8000000000000000 [126991.398732] mlx5_core 0000:04:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 0 8000000000000000 [133327.700369] LustreError: 185190:0:(ldlm_lockd.c:261:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 172.25.80.29@tcp ns: mdt-sfa18k03-MDT0001_UUID lock: 00000000f6de91b0/0xcf4e9953b5e914a4 lrc: 3/0,0 mode: PW/PW res: [0x28003bd45:0x2116:0x0].0x0 bits 0x49/0x0 rrc: 5 type: IBT gid 0 flags: 0x60200400000020 nid: 172.25.80.29@tcp remote: 0xb49974d9a1e374c5 expref: 81816 pid: 783733 timeout: 133325 lvb_type: 0 [133327.712817] LustreError: 604374:0:(ldlm_lib.c:3537:target_bulk_io()) @@@ Eviction on bulk WRITE req@000000008f547f41 x1816491036992832/t0(0) o4->c0a85f24-0848-494e-a322-757690560c78@172.25.80.29@tcp:482/0 lens 488/448 e 0 to 0 dl 1732351002 ref 1 fl Interpret:/0/0 rc 0/0 job:'' [133327.715957] LustreError: 604480:0:(tgt_grant.c:767:tgt_grant_check()) sfa18k03-MDT0001: cli c0a85f24-0848-494e-a322-757690560c78 claims 49152 GRANT, real grant 0 [133327.721996] Lustre: sfa18k03-MDT0001: Bulk IO write error with c0a85f24-0848-494e-a322-757690560c78 (at 172.25.80.29@tcp), client will retry: rc = -107 [133327.722000] LustreError: 604480:0:(ldlm_lib.c:3493:target_bulk_io()) @@@ bulk WRITE failed: rc = -107 req@0000000070c4588f x1816491036993216/t0(0) o4->c0a85f24-0848-494e-a322-757690560c78@172.25.80.29@tcp:482/0 lens 488/448 e 0 to 0 dl 1732351002 ref 1 fl Interpret:/0/0 rc 0/0 job:'' [133333.437312] LustreError: 783731:0:(client.c:1256:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@0000000071e2719c x1816365694466560/t0(0) o104->sfa18k03-MDT0001@172.25.80.29@tcp:15/16 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' [133333.740974] LustreError: 185209:0:(ldlm_lockd.c:1427:ldlm_handle_enqueue0()) ### lock on destroyed export 000000001c987727 ns: mdt-sfa18k03-MDT0001_UUID lock: 0000000091330282/0xcf4e9953b5e9171a lrc: 3/0,0 mode: PR/PR res: [0x28003bd45:0x2116:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x50200000000000 nid: 172.25.80.29@tcp remote: 0xb49974d9a1e37638 expref: 32409 pid: 185209 timeout: 0 lvb_type: 0 [133336.300887] LustreError: 783733:0:(client.c:1256:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@000000005d9bcd6f x1816365694587456/t0(0) o104->sfa18k03-MDT0001@172.25.80.29@tcp:15/16 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' [133336.306971] LustreError: 783733:0:(client.c:1256:ptlrpc_import_delay_req()) Skipped 169 previous similar messages [133337.513019] LustreError: 185209:0:(client.c:1256:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@0000000095a168e2 x1816365694681024/t0(0) o104->sfa18k03-MDT0001@172.25.80.29@tcp:15/16 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' [133337.518675] LustreError: 185209:0:(client.c:1256:ptlrpc_import_delay_req()) Skipped 157 previous similar messages [133339.698606] LustreError: 783733:0:(client.c:1256:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@0000000015f20f68 x1816365694761664/t0(0) o104->sfa18k03-MDT0001@172.25.80.29@tcp:15/16 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' [133339.704612] LustreError: 783733:0:(client.c:1256:ptlrpc_import_delay_req()) Skipped 106 previous similar messages [133343.732133] LustreError: 783731:0:(client.c:1256:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@00000000db78fffa x1816365694942912/t0(0) o104->sfa18k03-MDT0001@172.25.80.29@tcp:15/16 lens 328/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' [133343.738111] LustreError: 783731:0:(client.c:1256:ptlrpc_import_delay_req()) Skipped 129 previous similar messages [141634.029432] LustreError: 185190:0:(ldlm_lockd.c:261:expired_lock_main()) ### lock callback timer expired after 101s: evicting client at 172.25.80.23@tcp ns: mdt-sfa18k03-MDT0001_UUID lock: 00000000b5154d9a/0xcf4e9953b9b13a47 lrc: 3/0,0 mode: PW/PW res: [0x28003bd43:0x1835:0x0].0x0 bits 0x41/0x0 rrc: 13 type: IBT gid 0 flags: 0x60200400000020 nid: 172.25.80.23@tcp remote: 0x5876d76d8f153bac expref: 4430 pid: 532338 timeout: 141630 lvb_type: 0 [141634.048786] LustreError: 531264:0:(client.c:1256:ptlrpc_import_delay_req()) @@@ IMP_CLOSED req@000000001b0bba5a x1816366164542528/t0(0) o105->sfa18k03-MDT0001@172.25.80.23@tcp:15/16 lens 392/224 e 0 to 0 dl 0 ref 1 fl Rpc:QU/0/ffffffff rc 0/-1 job:'' [141634.054725] LustreError: 531264:0:(client.c:1256:ptlrpc_import_delay_req()) Skipped 12 previous similar messages [141634.057561] LustreError: 185206:0:(ldlm_lockd.c:1427:ldlm_handle_enqueue0()) ### lock on destroyed export 000000005a64d35a ns: mdt-sfa18k03-MDT0001_UUID lock: 000000007b0cd201/0xcf4e9953b9b16d77 lrc: 3/0,0 mode: PR/PR res: [0x28003bd43:0x1835:0x0].0x0 bits 0x20/0x0 rrc: 10 type: IBT gid 0 flags: 0x50200000000000 nid: 172.25.80.23@tcp remote: 0x5876d76d8f155368 expref: 2867 pid: 185206 timeout: 0 lvb_type: 0 [142503.942147] mlx5_core 0000:03:00.0: temp_warn:173:(pid 121123): High temperature on sensors with bit set 1 8000000000000000 [142503.957525] mlx5_core 0000:04:00.0: temp_warn:173:(pid 121118): High temperature on sensors with bit set 1 8000000000000000 [143084.962766] mlx5_core 0000:03:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 0 8000000000000000 [143084.976828] mlx5_core 0000:04:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 0 8000000000000000 [146680.394833] LustreError: 185190:0:(ldlm_lockd.c:261:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 172.25.80.27@tcp ns: mdt-sfa18k03-MDT0001_UUID lock: 00000000b2c13a26/0xcf4e9953c13632e5 lrc: 3/0,0 mode: PW/PW res: [0x28003bd4a:0xf5c0:0x0].0x0 bits 0x41/0x0 rrc: 7 type: IBT gid 0 flags: 0x60200400000020 nid: 172.25.80.27@tcp remote: 0x730f604d3ba04737 expref: 43 pid: 531746 timeout: 146677 lvb_type: 0 [146680.404978] LustreError: 185203:0:(ldlm_lockd.c:1427:ldlm_handle_enqueue0()) ### lock on destroyed export 000000002078a64e ns: mdt-sfa18k03-MDT0001_UUID lock: 00000000cf0ef0f5/0xcf4e9953c13772ed lrc: 3/0,0 mode: PR/PR res: [0x28003bd4a:0xf5c0:0x0].0x0 bits 0x20/0x0 rrc: 2 type: IBT gid 0 flags: 0x50200000000000 nid: 172.25.80.27@tcp remote: 0x730f604d3ba06322 expref: 9 pid: 185203 timeout: 0 lvb_type: 0 [146681.574627] Lustre: 527403:0:(mdt_handler.c:3081:mdt_object_sync()) sfa18k03-MDT0001: non existing object [0x28003bd4a:0xf5d1:0x0]: rc = -116 [150459.535825] LustreError: 185190:0:(ldlm_lockd.c:261:expired_lock_main()) ### lock callback timer expired after 100s: evicting client at 172.25.80.26@tcp ns: filter-sfa18k03-OST0022_UUID lock: 00000000f5e61a38/0xcf4e9953c2e12a17 lrc: 3/0,0 mode: PW/PW res: [0xb80000bd0:0x1d47f5c:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 1048576->18446744073709551615) gid 0 flags: 0x60000480020020 nid: 172.25.80.26@tcp remote: 0x6e1e7c450a62f132 expref: 158 pid: 527605 timeout: 150456 lvb_type: 0 [151205.255233] mlx5_core 0000:03:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 1 8000000000000000 [151205.266558] mlx5_core 0000:04:00.0: temp_warn:173:(pid 0): High temperature on sensors with bit set 1 8000000000000000